This is an automated email from the ASF dual-hosted git repository.
prashantpogde pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f4b6bb1e52 HDDS-8739. Snapdiff should return complete absolute path in
Diff Entry (#4823)
f4b6bb1e52 is described below
commit f4b6bb1e52da23eb2bb86320073468e9e1191ec5
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Thu Jun 22 09:35:06 2023 -0700
HDDS-8739. Snapdiff should return complete absolute path in Diff Entry
(#4823)
---
.../org/apache/ozone/rocksdiff/RocksDiffUtils.java | 2 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 7 +-
.../ozone/snapshot/SnapshotDiffReportOzone.java | 46 +++++-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 6 +-
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 48 +++---
.../om/snapshot/FSODirectoryPathResolver.java | 109 +++++++++++++
.../ozone/om/snapshot/ObjectPathResolver.java | 34 ++++
.../ozone/om/snapshot/SnapshotDiffManager.java | 178 +++++++++++++++------
.../om/snapshot/TestFSODirectoryPathResolver.java | 150 +++++++++++++++++
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 52 +++---
10 files changed, 531 insertions(+), 101 deletions(-)
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
index cca3eaf0e7..225130bce7 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
@@ -89,7 +89,7 @@ public final class RocksDiffUtils {
TableProperties properties = sstFileReader.get().getTableProperties();
String tableName = new String(properties.getColumnFamilyName(), UTF_8);
if (tableToPrefixMap.containsKey(tableName)) {
- String prefix = tableToPrefixMap.get(tableName) + OM_KEY_PREFIX;
+ String prefix = tableToPrefixMap.get(tableName);
try (ManagedSstFileReaderIterator iterator =
ManagedSstFileReaderIterator.managed(sstFileReader.get()
.newIterator(new ReadOptions()))) {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 4eea2aa7a2..1ff24bed16 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -40,7 +40,6 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.Comparator;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdds.conf.ConfigurationException;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
@@ -65,6 +64,7 @@ import static
org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BIND_HOST_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DECOMMISSIONED_NODES_KEY;
@@ -675,7 +675,6 @@ public final class OmUtils {
* does not preserve.
* @return normalized key name.
*/
- @SuppressFBWarnings("DMI_HARDCODED_ABSOLUTE_FILENAME")
public static String normalizeKey(String keyName,
boolean preserveTrailingSlash) {
// For empty strings do nothing, just return the same.
@@ -692,8 +691,8 @@ public final class OmUtils {
LOG.debug("Normalized key {} to {} ", keyName,
normalizedKeyName.substring(1));
}
- if (preserveTrailingSlash && keyName.endsWith("/")) {
- return normalizedKeyName.substring(1) + "/";
+ if (preserveTrailingSlash && keyName.endsWith(OZONE_URI_DELIMITER)) {
+ return normalizedKeyName.substring(1) + OZONE_URI_DELIMITER;
}
return normalizedKeyName.substring(1);
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
index 03a1ed418e..14f204999f 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
@@ -24,12 +24,15 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.db.Codec;
import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
import org.apache.hadoop.hdds.utils.db.Proto2Codec;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DiffReportEntryProto;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffReportProto;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
import java.util.List;
import java.util.stream.Collectors;
@@ -163,7 +166,7 @@ public class SnapshotDiffReportOzone
return null;
}
DiffType type = fromProtobufDiffType(entry.getDiffType());
- return type == null ? null : new DiffReportEntry(type,
+ return type == null ? null : new DiffReportEntryOzone(type,
entry.getSourcePath().getBytes(StandardCharsets.UTF_8),
entry.hasTargetPath() ?
entry.getTargetPath().getBytes(StandardCharsets.UTF_8) : null);
@@ -191,7 +194,7 @@ public class SnapshotDiffReportOzone
public static DiffReportEntry getDiffReportEntry(final DiffType type,
final String sourcePath, final String targetPath) {
- return new DiffReportEntry(type,
+ return new DiffReportEntryOzone(type,
sourcePath.getBytes(StandardCharsets.UTF_8),
targetPath != null ? targetPath.getBytes(StandardCharsets.UTF_8) :
null);
@@ -206,5 +209,44 @@ public class SnapshotDiffReportOzone
this.getDiffList().addAll(diffReport.getDiffList());
}
+ /**
+ * DiffReportEntry for ozone.
+ */
+ public static class DiffReportEntryOzone extends DiffReportEntry {
+
+ public DiffReportEntryOzone(DiffType type, byte[] sourcePath) {
+ super(type, sourcePath);
+ }
+
+ public DiffReportEntryOzone(DiffType type, byte[][] sourcePathComponents) {
+ super(type, sourcePathComponents);
+ }
+
+ public DiffReportEntryOzone(DiffType type, byte[] sourcePath,
+ byte[] targetPath) {
+ super(type, sourcePath, targetPath);
+ }
+ public DiffReportEntryOzone(DiffType type, byte[][] sourcePathComponents,
+ byte[][] targetPathComponents) {
+ super(type, sourcePathComponents, targetPathComponents);
+ }
+
+ static String getPathString(byte[] path) {
+ String pathStr = DFSUtilClient.bytes2String(path);
+ return pathStr.isEmpty() ? "." : Paths.get(pathStr)
+ .toAbsolutePath().toString();
+ }
+
+ @Override
+ public String toString() {
+ String str = this.getType().getLabel() + "\t" +
+ getPathString(this.getSourcePath());
+ if (this.getType() == SnapshotDiffReport.DiffType.RENAME) {
+ str = str + " -> " + getPathString(this.getTargetPath());
+ }
+
+ return str;
+ }
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index 4ea79970f2..6f2be4b099 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -2481,9 +2481,11 @@ public class TestRootedOzoneFileSystem {
diff.getDiffList().get(0).getType());
Assert.assertEquals(SnapshotDiffReport.DiffType.CREATE,
diff.getDiffList().get(1).getType());
- Assert.assertArrayEquals("key1".getBytes(StandardCharsets.UTF_8),
+ Assert.assertArrayEquals(
+ "/key1".getBytes(StandardCharsets.UTF_8),
diff.getDiffList().get(0).getSourcePath());
- Assert.assertArrayEquals("key2".getBytes(StandardCharsets.UTF_8),
+ Assert.assertArrayEquals(
+ "/key2".getBytes(StandardCharsets.UTF_8),
diff.getDiffList().get(1).getSourcePath());
// test whether snapdiff returns aggregated response as
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index 1d94ba1b6a..4195fa0994 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils;
import org.apache.hadoop.hdds.utils.db.DBProfile;
import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.TestDataUtil;
@@ -81,6 +82,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static
org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isDone;
import static
org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isStarting;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
@@ -216,7 +218,7 @@ public class TestOmSnapshot {
// stop the deletion services so that keys can still be read
keyManager.stop();
- preFinalizationChecks();
+// preFinalizationChecks();
finalizeOMUpgrade();
counter = new AtomicInteger();
}
@@ -599,12 +601,14 @@ public class TestOmSnapshot {
SnapshotDiffReportOzone
diff2 = getSnapDiffReport(volume, bucket, snap2, snap3);
Assert.assertEquals(2, diff2.getDiffList().size());
- Assert.assertTrue(diff2.getDiffList().contains(
- SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReportOzone.DiffType.CREATE, key2)));
- Assert.assertTrue(diff2.getDiffList().contains(
- SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReportOzone.DiffType.DELETE, key1)));
+ Assert.assertEquals(
+ Arrays.asList(SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE,
+ OZONE_URI_DELIMITER + key1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE,
+ OZONE_URI_DELIMITER + key2)),
+ diff2.getDiffList());
// Rename Key2
String key2Renamed = key2 + "_renamed";
@@ -617,7 +621,8 @@ public class TestOmSnapshot {
Assert.assertEquals(1, diff3.getDiffList().size());
Assert.assertTrue(diff3.getDiffList().contains(
SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReportOzone.DiffType.RENAME, key2, key2Renamed)));
+ SnapshotDiffReportOzone.DiffType.RENAME, OZONE_URI_DELIMITER +
key2,
+ OZONE_URI_DELIMITER + key2Renamed)));
// Create a directory
@@ -628,14 +633,10 @@ public class TestOmSnapshot {
SnapshotDiffReportOzone
diff4 = getSnapDiffReport(volume, bucket, snap4, snap5);
Assert.assertEquals(1, diff4.getDiffList().size());
- // for non-fso, directories are a special type of key with "/" appended
- // at the end.
- if (!bucket1.getBucketLayout().isFileSystemOptimized()) {
- dir1 = dir1 + OM_KEY_PREFIX;
- }
Assert.assertTrue(diff4.getDiffList().contains(
SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReportOzone.DiffType.CREATE, dir1)));
+ SnapshotDiffReportOzone.DiffType.CREATE,
+ OM_KEY_PREFIX + dir1)));
String key3 = createFileKeyWithPrefix(bucket1, "key-3-");
String snap6 = "snap" + counter.incrementAndGet();
@@ -648,17 +649,14 @@ public class TestOmSnapshot {
createSnapshot(volume, bucket, snap7);
SnapshotDiffReportOzone
diff5 = getSnapDiffReport(volume, bucket, snap6, snap7);
- assertEquals(2, diff5.getDiffList().size());
- assertEquals(SnapshotDiffReportOzone.DiffType.RENAME,
- diff5.getDiffList().get(0).getType());
- assertEquals(key3, org.apache.hadoop.hdds.StringUtils.bytes2String(
- diff5.getDiffList().get(0).getSourcePath()));
- assertEquals(renamedKey3, org.apache.hadoop.hdds.StringUtils.bytes2String(
- diff5.getDiffList().get(0).getTargetPath()));
- assertEquals(SnapshotDiffReportOzone.DiffType.MODIFY,
- diff5.getDiffList().get(1).getType());
- assertEquals(key3, org.apache.hadoop.hdds.StringUtils.bytes2String(
- diff5.getDiffList().get(1).getSourcePath()));
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffList =
+ Arrays.asList(SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.RENAME, OZONE_URI_DELIMITER + key3,
+ OZONE_URI_DELIMITER + renamedKey3),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.MODIFY, OZONE_URI_DELIMITER + key3)
+ );
+ assertEquals(expectedDiffList, diff5.getDiffList());
}
@Test
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/FSODirectoryPathResolver.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/FSODirectoryPathResolver.java
new file mode 100644
index 0000000000..0d0bcb2276
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/FSODirectoryPathResolver.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.snapshot;
+
+import com.google.common.collect.Sets;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.Set;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
+/**
+ * Class to resolve absolute paths for FSO DirectoryInfo Objects.
+ */
+public class FSODirectoryPathResolver implements ObjectPathResolver {
+
+ private final String prefix;
+ private final long bucketId;
+ private final Table<String, OmDirectoryInfo> dirInfoTable;
+
+ public FSODirectoryPathResolver(String prefix, long bucketId,
+ Table<String, OmDirectoryInfo> dirInfoTable) {
+ this.prefix = prefix;
+ this.dirInfoTable = dirInfoTable;
+ this.bucketId = bucketId;
+ }
+
+ private void addToPathMap(Pair<Long, Path> objectIDPath,
+ Set<Long> dirObjIds, Map<Long, Path> pathMap) {
+ if (dirObjIds.contains(objectIDPath.getKey())) {
+ pathMap.put(objectIDPath.getKey(), objectIDPath.getValue());
+ dirObjIds.remove(objectIDPath.getKey());
+ }
+ }
+
+ /**
+ * Assuming all dirObjIds belong to a bucket this function resolves absolute
+ * path for a given FSO bucket.
+ * @param dirObjIds Object Ids corresponding to which absolute path is
needed.
+ * @return Map of Path corresponding to provided directory object IDs
+ */
+ @SuppressFBWarnings("DMI_HARDCODED_ABSOLUTE_FILENAME")
+ @Override
+ public Map<Long, Path> getAbsolutePathForObjectIDs(
+ Optional<Set<Long>> dirObjIds) throws IOException {
+ // Root of a bucket would always have the
+ // key as /volumeId/bucketId/bucketId/
+ if (!dirObjIds.isPresent() || dirObjIds.get().isEmpty()) {
+ return Collections.emptyMap();
+ }
+ Set<Long> objIds = Sets.newHashSet(dirObjIds.get());
+ Map<Long, Path> objectIdPathMap = new HashMap<>();
+ Queue<Pair<Long, Path>> objectIdPathVals = new LinkedList<>();
+ Pair<Long, Path> root = Pair.of(bucketId, Paths.get(OZONE_URI_DELIMITER));
+ objectIdPathVals.add(root);
+ addToPathMap(root, objIds, objectIdPathMap);
+
+ while (!objectIdPathVals.isEmpty() && objIds.size() > 0) {
+ Pair<Long, Path> parent = objectIdPathVals.poll();
+ try (TableIterator<String,
+ ? extends Table.KeyValue<String, OmDirectoryInfo>>
+ subDirIter = dirInfoTable.iterator(
+ prefix + parent.getKey() + OM_KEY_PREFIX)) {
+ while (objIds.size() > 0 && subDirIter.hasNext()) {
+ OmDirectoryInfo childDir = subDirIter.next().getValue();
+ Pair<Long, Path> pathVal = Pair.of(childDir.getObjectID(),
+ parent.getValue().resolve(childDir.getName()));
+ addToPathMap(pathVal, objIds, objectIdPathMap);
+ objectIdPathVals.add(pathVal);
+ }
+ }
+ }
+ // Invalid directory objectId which does not exist in the given bucket.
+ if (objIds.size() > 0) {
+ throw new IllegalArgumentException(
+ "Dir object Ids required but not found in bucket: " + objIds);
+ }
+ return objectIdPathMap;
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/ObjectPathResolver.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/ObjectPathResolver.java
new file mode 100644
index 0000000000..b9c5bb0c45
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/ObjectPathResolver.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Class to resolve paths of Objects.
+ */
+public interface ObjectPathResolver {
+
+ Map<Long, Path> getAbsolutePathForObjectIDs(Optional<Set<Long>> objIds)
+ throws IOException;
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
index 9ad256a016..d4a759e2cd 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.om.snapshot;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.BufferedWriter;
@@ -41,6 +42,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException;
@@ -61,6 +63,7 @@ import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool;
import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
@@ -74,6 +77,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.WithObjectID;
import org.apache.hadoop.ozone.om.service.SnapshotDeletingService;
import
org.apache.hadoop.ozone.om.snapshot.SnapshotDiffObject.SnapshotDiffObjectBuilder;
+import org.apache.hadoop.ozone.om.helpers.WithParentObjectId;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.util.ClosableIterator;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
@@ -94,6 +98,7 @@ import org.slf4j.LoggerFactory;
import java.util.concurrent.SynchronousQueue;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_JOB_DEFAULT_WAIT_TIME;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_JOB_DEFAULT_WAIT_TIME_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_MAX_ALLOWED_KEYS_CHANGED_PER_DIFF_JOB;
@@ -106,7 +111,6 @@ import static
org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER;
import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.getTableKey;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotActive;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.dropColumnFamilyHandle;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getSnapshotInfo;
import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.CANCELLED;
import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
@@ -343,11 +347,12 @@ public class SnapshotDiffManager implements AutoCloseable
{
String bucketId = String.valueOf(
omMetadataManager.getBucketId(volumeName, bucketName));
tablePrefixes.put(OmMetadataManagerImpl.KEY_TABLE,
- OM_KEY_PREFIX + volumeName + OM_KEY_PREFIX + bucketName);
+ OM_KEY_PREFIX + volumeName + OM_KEY_PREFIX + bucketName
+ + OM_KEY_PREFIX);
tablePrefixes.put(OmMetadataManagerImpl.FILE_TABLE,
- OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId);
+ OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId + OM_KEY_PREFIX);
tablePrefixes.put(OmMetadataManagerImpl.DIRECTORY_TABLE,
- OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId);
+ OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId + OM_KEY_PREFIX);
return tablePrefixes;
}
@@ -859,6 +864,25 @@ public class SnapshotDiffManager implements AutoCloseable {
Table<String, OmKeyInfo> tsKeyTable = toSnapshot.getMetadataManager()
.getKeyTable(bucketLayout);
+ final Optional<Set<Long>> oldParentIds;
+ final Optional<Set<Long>> newParentIds;
+ if (bucketLayout.isFileSystemOptimized()) {
+ oldParentIds = Optional.of(new HashSet<>());
+ newParentIds = Optional.of(new HashSet<>());
+ } else {
+ oldParentIds = Optional.empty();
+ newParentIds = Optional.empty();
+ }
+
+ final Optional<Map<Long, Path>> oldParentIdPathMap;
+ final Optional<Map<Long, Path>> newParentIdPathMap;
+ if (bucketLayout.isFileSystemOptimized()) {
+ oldParentIdPathMap = Optional.of(Maps.newHashMap());
+ newParentIdPathMap = Optional.of(Maps.newHashMap());
+ } else {
+ oldParentIdPathMap = Optional.empty();
+ newParentIdPathMap = Optional.empty();
+ }
// These are the most time and resource consuming method calls.
// Split the calls into steps and store them in an array, to avoid
// repetition while constantly checking if the job is cancelled.
@@ -868,7 +892,7 @@ public class SnapshotDiffManager implements AutoCloseable {
fromSnapshot, toSnapshot, fsInfo, tsInfo, useFullDiff,
tablePrefixes, objectIdToKeyNameMapForFromSnapshot,
objectIdToKeyNameMapForToSnapshot, objectIdToDiffObject,
- path.toString());
+ oldParentIds, newParentIds, path.toString());
return null;
},
() -> {
@@ -882,7 +906,25 @@ public class SnapshotDiffManager implements AutoCloseable {
fromSnapshot, toSnapshot, fsInfo, tsInfo, useFullDiff,
tablePrefixes, objectIdToKeyNameMapForFromSnapshot,
objectIdToKeyNameMapForToSnapshot, objectIdToDiffObject,
- path.toString());
+ oldParentIds, newParentIds, path.toString());
+ }
+ return null;
+ },
+ () -> {
+ if (bucketLayout.isFileSystemOptimized()) {
+ long bucketId = toSnapshot.getMetadataManager()
+ .getBucketId(volumeName, bucketName);
+ String tablePrefix = getTablePrefix(tablePrefixes,
+ fromSnapshot.getMetadataManager()
+ .getDirectoryTable().getName());
+ oldParentIdPathMap.get().putAll(new FSODirectoryPathResolver(
+ tablePrefix, bucketId,
+ fromSnapshot.getMetadataManager().getDirectoryTable())
+ .getAbsolutePathForObjectIDs(oldParentIds));
+ newParentIdPathMap.get().putAll(new FSODirectoryPathResolver(
+ tablePrefix, bucketId,
+ toSnapshot.getMetadataManager().getDirectoryTable())
+ .getAbsolutePathForObjectIDs(newParentIds));
}
return null;
},
@@ -894,7 +936,9 @@ public class SnapshotDiffManager implements AutoCloseable {
objectIdToKeyNameMapForFromSnapshot,
objectIdToKeyNameMapForToSnapshot,
volumeName, bucketName,
- fromSnapshotName, toSnapshotName);
+ fromSnapshotName, toSnapshotName,
+ bucketLayout.isFileSystemOptimized(), oldParentIdPathMap,
+ newParentIdPathMap);
// If job is cancelled, totalDiffEntries will be equal to -1.
if (totalDiffEntries >= 0 &&
areDiffJobAndSnapshotsActive(volumeName, bucketName,
@@ -939,19 +983,17 @@ public class SnapshotDiffManager implements AutoCloseable
{
@SuppressWarnings("checkstyle:ParameterNumber")
private void getDeltaFilesAndDiffKeysToObjectIdToKeyMap(
- final Table<String, ? extends WithObjectID> fsTable,
- final Table<String, ? extends WithObjectID> tsTable,
- final OmSnapshot fromSnapshot,
- final OmSnapshot toSnapshot,
- final SnapshotInfo fsInfo,
- final SnapshotInfo tsInfo,
- final boolean useFullDiff,
- final Map<String, String> tablePrefixes,
+ final Table<String, ? extends WithParentObjectId> fsTable,
+ final Table<String, ? extends WithParentObjectId> tsTable,
+ final OmSnapshot fromSnapshot, final OmSnapshot toSnapshot,
+ final SnapshotInfo fsInfo, final SnapshotInfo tsInfo,
+ final boolean useFullDiff, final Map<String, String> tablePrefixes,
final PersistentMap<byte[], byte[]> oldObjIdToKeyMap,
final PersistentMap<byte[], byte[]> newObjIdToKeyMap,
final PersistentMap<byte[], SnapshotDiffObject> objectIdToDiffObject,
- final String diffDir
- ) throws IOException, RocksDBException {
+ final Optional<Set<Long>> oldParentIds,
+ final Optional<Set<Long>> newParentIds,
+ final String diffDir) throws IOException, RocksDBException {
List<String> tablesToLookUp = Collections.singletonList(fsTable.getName());
@@ -974,6 +1016,8 @@ public class SnapshotDiffManager implements AutoCloseable {
oldObjIdToKeyMap,
newObjIdToKeyMap,
objectIdToDiffObject,
+ oldParentIds,
+ newParentIds,
tablePrefixes);
} catch (NativeLibraryNotLoadedException e) {
LOG.warn("SSTDumpTool load failure, retrying without it.", e);
@@ -990,6 +1034,8 @@ public class SnapshotDiffManager implements AutoCloseable {
oldObjIdToKeyMap,
newObjIdToKeyMap,
objectIdToDiffObject,
+ oldParentIds,
+ newParentIds,
tablePrefixes);
} catch (NativeLibraryNotLoadedException ex) {
throw new IllegalStateException(ex);
@@ -998,19 +1044,20 @@ public class SnapshotDiffManager implements
AutoCloseable {
}
@SuppressWarnings("checkstyle:ParameterNumber")
- void addToObjectIdMap(
- final Table<String, ? extends WithObjectID> fsTable,
- final Table<String, ? extends WithObjectID> tsTable,
- final Set<String> deltaFiles, boolean nativeRocksToolsLoaded,
- final PersistentMap<byte[], byte[]> oldObjIdToKeyMap,
- final PersistentMap<byte[], byte[]> newObjIdToKeyMap,
+ void addToObjectIdMap(Table<String, ? extends WithParentObjectId> fsTable,
+ Table<String, ? extends WithParentObjectId> tsTable,
+ Set<String> deltaFiles, boolean nativeRocksToolsLoaded,
+ PersistentMap<byte[], byte[]> oldObjIdToKeyMap,
+ PersistentMap<byte[], byte[]> newObjIdToKeyMap,
final PersistentMap<byte[], SnapshotDiffObject> objectIdToDiffObject,
- final Map<String, String> tablePrefixes
- ) throws IOException,
+ Optional<Set<Long>> oldParentIds,
+ Optional<Set<Long>> newParentIds,
+ Map<String, String> tablePrefixes) throws IOException,
NativeLibraryNotLoadedException, RocksDBException {
if (deltaFiles.isEmpty()) {
return;
}
+ String tablePrefix = getTablePrefix(tablePrefixes, fsTable.getName());
boolean isDirectoryTable =
fsTable.getName().equals(OmMetadataManagerImpl.DIRECTORY_TABLE);
ManagedSstFileReader sstFileReader = new ManagedSstFileReader(deltaFiles);
@@ -1022,8 +1069,8 @@ public class SnapshotDiffManager implements AutoCloseable
{
: sstFileReader.getKeyStream()) {
keysToCheck.forEach(key -> {
try {
- final WithObjectID fromObjectId = fsTable.get(key);
- final WithObjectID toObjectId = tsTable.get(key);
+ final WithParentObjectId fromObjectId = fsTable.get(key);
+ final WithParentObjectId toObjectId = tsTable.get(key);
if (areKeysEqual(fromObjectId, toObjectId) || !isKeyInBucket(key,
tablePrefixes, fsTable.getName())) {
// We don't have to do anything.
@@ -1032,23 +1079,31 @@ public class SnapshotDiffManager implements
AutoCloseable {
if (fromObjectId != null) {
byte[] rawObjId = codecRegistry.asRawData(
fromObjectId.getObjectID());
+ // Removing volume bucket info by removing the table bucket Prefix
+ // from the key.
+ // For FSO buckets will be left with the parent id/keyname.
+ // For OBS buckets will be left with the complete path
byte[] rawValue = codecRegistry.asRawData(
- getKeyOrDirectoryName(isDirectoryTable, fromObjectId));
+ key.substring(tablePrefix.length()));
oldObjIdToKeyMap.put(rawObjId, rawValue);
SnapshotDiffObject diffObject =
createDiffObjectWithOldName(fromObjectId.getObjectID(), key,
- objectIdToDiffObject.get(rawObjId));
+ objectIdToDiffObject.get(rawObjId));
objectIdToDiffObject.put(rawObjId, diffObject);
+ oldParentIds.ifPresent(set -> set.add(
+ fromObjectId.getParentObjectID()));
}
if (toObjectId != null) {
byte[] rawObjId =
codecRegistry.asRawData(toObjectId.getObjectID());
byte[] rawValue = codecRegistry.asRawData(
- getKeyOrDirectoryName(isDirectoryTable, toObjectId));
+ key.substring(tablePrefix.length()));
newObjIdToKeyMap.put(rawObjId, rawValue);
SnapshotDiffObject diffObject =
createDiffObjectWithNewName(toObjectId.getObjectID(), key,
objectIdToDiffObject.get(rawObjId));
objectIdToDiffObject.put(rawObjId, diffObject);
+ newParentIds.ifPresent(set -> set.add(toObjectId
+ .getParentObjectID()));
}
} catch (IOException e) {
throw new RuntimeException(e);
@@ -1159,6 +1214,25 @@ public class SnapshotDiffManager implements
AutoCloseable {
}
}
+ @SuppressFBWarnings("DMI_HARDCODED_ABSOLUTE_FILENAME")
+ private String resolveAbsolutePath(boolean isFSOBucket,
+ final Optional<Map<Long, Path>> parentIdMap, byte[] keyVal)
+ throws IOException {
+ String key = codecRegistry.asObject(keyVal, String.class);
+ if (isFSOBucket) {
+ String[] splitKey = key.split(OM_KEY_PREFIX, 2);
+ Long parentId = Long.valueOf(splitKey[0]);
+ if (parentIdMap.map(m -> !m.containsKey(parentId)).orElse(true)) {
+ throw new IllegalStateException(String.format(
+ "Cannot resolve path for key: %s with parent Id: %d", key,
+ parentId));
+ }
+ return parentIdMap.map(m -> m.get(parentId).resolve(splitKey[1]))
+ .get().toString();
+ }
+ return Paths.get(OzoneConsts.OZONE_URI_DELIMITER).resolve(key).toString();
+ }
+
@SuppressWarnings({"checkstyle:ParameterNumber", "checkstyle:MethodLength"})
long generateDiffReport(
final String jobId,
@@ -1170,7 +1244,10 @@ public class SnapshotDiffManager implements
AutoCloseable {
final String volumeName,
final String bucketName,
final String fromSnapshotName,
- final String toSnapshotName) {
+ final String toSnapshotName,
+ final boolean isFSOBucket,
+ final Optional<Map<Long, Path>> oldParentIdPathMap,
+ final Optional<Map<Long, Path>> newParentIdPathMap) {
LOG.info("Starting diff report generation for jobId: {}.", jobId);
ColumnFamilyHandle deleteDiffColumnFamily = null;
ColumnFamilyHandle renameDiffColumnFamily = null;
@@ -1236,26 +1313,31 @@ public class SnapshotDiffManager implements
AutoCloseable {
throw new IllegalStateException(
"Old and new key name both are null");
} else if (oldKeyName == null) { // Key Created.
- String key = codecRegistry.asObject(newKeyName, String.class);
+ String key = resolveAbsolutePath(isFSOBucket, newParentIdPathMap,
+ newKeyName);
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(DiffType.CREATE,
key);
createDiffs.add(codecRegistry.asRawData(entry));
} else if (newKeyName == null) { // Key Deleted.
- String key = codecRegistry.asObject(oldKeyName, String.class);
+ String key = resolveAbsolutePath(isFSOBucket, oldParentIdPathMap,
+ oldKeyName);
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(DiffType.DELETE,
key);
deleteDiffs.add(codecRegistry.asRawData(entry));
} else if (Arrays.equals(oldKeyName, newKeyName)) { // Key modified.
- String key = codecRegistry.asObject(newKeyName, String.class);
+ String key = resolveAbsolutePath(isFSOBucket, newParentIdPathMap,
+ newKeyName);
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(DiffType.MODIFY,
key);
modifyDiffs.add(codecRegistry.asRawData(entry));
- } else {
- String oldKey = codecRegistry.asObject(oldKeyName, String.class);
- String newKey = codecRegistry.asObject(newKeyName, String.class);
+ } else { // Key Renamed.
+ String oldKey = resolveAbsolutePath(isFSOBucket,
oldParentIdPathMap,
+ oldKeyName);
+ String newKey = resolveAbsolutePath(isFSOBucket,
newParentIdPathMap,
+ newKeyName);
renameDiffs.add(codecRegistry.asRawData(
SnapshotDiffReportOzone.getDiffReportEntry(DiffType.RENAME,
oldKey, newKey)));
@@ -1440,23 +1522,27 @@ public class SnapshotDiffManager implements
AutoCloseable {
}
/**
- * check if the given key is in the bucket specified by tablePrefix map.
+ * Get table prefix given a tableName.
*/
- boolean isKeyInBucket(String key, Map<String, String> tablePrefixes,
- String tableName) {
- String volumeBucketDbPrefix;
+ private String getTablePrefix(Map<String, String> tablePrefixes,
+ String tableName) {
// In case of FSO - either File/Directory table
// the key Prefix would be volumeId/bucketId and
// in case of non-fso - volumeName/bucketName
if (tableName.equals(
OmMetadataManagerImpl.DIRECTORY_TABLE) || tableName.equals(
OmMetadataManagerImpl.FILE_TABLE)) {
- volumeBucketDbPrefix =
- tablePrefixes.get(OmMetadataManagerImpl.DIRECTORY_TABLE);
- } else {
- volumeBucketDbPrefix =
tablePrefixes.get(OmMetadataManagerImpl.KEY_TABLE);
+ return tablePrefixes.get(OmMetadataManagerImpl.DIRECTORY_TABLE);
}
- return key.startsWith(volumeBucketDbPrefix);
+ return tablePrefixes.get(OmMetadataManagerImpl.KEY_TABLE);
+ }
+
+ /**
+ * check if the given key is in the bucket specified by tablePrefix map.
+ */
+ boolean isKeyInBucket(String key, Map<String, String> tablePrefixes,
+ String tableName) {
+ return key.startsWith(getTablePrefix(tablePrefixes, tableName));
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestFSODirectoryPathResolver.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestFSODirectoryPathResolver.java
new file mode 100644
index 0000000000..f56b8657e4
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestFSODirectoryPathResolver.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.snapshot;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+
+/**
+ * Test class for FSODirectoryPathResolver.
+ */
+public class TestFSODirectoryPathResolver {
+
+ private Table<String, OmDirectoryInfo> getMockedDirectoryInfoTable(
+ String prefix, Map<Integer, List<Integer>> dirMap) throws IOException {
+ Table<String, OmDirectoryInfo> dirInfos = Mockito.mock(Table.class);
+
+ Mockito.when(dirInfos.iterator(Mockito.anyString()))
+ .thenAnswer(i -> {
+ int dirId = Integer.parseInt(((String)i.getArgument(0))
+ .split(OM_KEY_PREFIX)[3]);
+ Iterator<? extends Table.KeyValue<String, OmDirectoryInfo>> iterator
=
+ dirMap
+ .getOrDefault(dirId, Collections.emptyList()).stream()
+ .map(children -> new Table.KeyValue<String, OmDirectoryInfo>() {
+ @Override
+ public String getKey() {
+ return prefix + children + OM_KEY_PREFIX + "dir" +
children;
+ }
+
+ @Override
+ public OmDirectoryInfo getValue() {
+ return OmDirectoryInfo.newBuilder()
+ .setName("dir" + children).setObjectID(children)
+ .build();
+ }
+ })
+ .iterator();
+ return new TableIterator<String,
+ Table.KeyValue<String, OmDirectoryInfo>>() {
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public Table.KeyValue<String, OmDirectoryInfo> next() {
+ return iterator.next();
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public void seekToFirst() {
+ }
+
+ @Override
+ public void seekToLast() {
+ }
+
+ @Override
+ public Table.KeyValue<String, OmDirectoryInfo> seek(String s) {
+ return null;
+ }
+
+ @Override
+ public void removeFromDB() {
+
+ }
+ };
+
+ });
+
+ return dirInfos;
+ }
+
+ @SuppressFBWarnings("DMI_HARDCODED_ABSOLUTE_FILENAME")
+ @Test
+ public void testGetAbsolutePathForValidObjectIDs() throws IOException {
+ Map<Integer, List<Integer>> dirMap = ImmutableMap.of(
+ 1, Lists.newArrayList(2, 3, 4, 5, 6),
+ 2, Lists.newArrayList(7, 8, 9, 10, 11),
+ 3, Lists.newArrayList(12, 13, 14, 15),
+ 9, Lists.newArrayList(16),
+ 14, Lists.newArrayList(17),
+ 18, Lists.newArrayList(19, 20)
+ );
+ String prefix = "/vol/buck/";
+ FSODirectoryPathResolver fsoDirectoryPathResolver =
+ new FSODirectoryPathResolver(prefix, 1,
+ getMockedDirectoryInfoTable(prefix, dirMap));
+ Set<Long> objIds = Sets.newHashSet(17L, 9L, 10L, 15L, 4L, 3L, 1L);
+ Map<Long, Path> absolutePathMap = fsoDirectoryPathResolver
+ .getAbsolutePathForObjectIDs(Optional.of(objIds));
+
+ Assertions.assertEquals(ImmutableMap.of(
+ 17L, Paths.get("/dir3/dir14/dir17"),
+ 9L, Paths.get("/dir2/dir9"),
+ 10L, Paths.get("/dir2/dir10"),
+ 15L, Paths.get("/dir3/dir15"),
+ 4L, Paths.get("/dir4"),
+ 3L, Paths.get("/dir3"),
+ 1L, Paths.get("/")
+ ), absolutePathMap);
+ Assertions.assertEquals(objIds.size(), absolutePathMap.size());
+ // Invalid Obj Id 19 with dirInfo dir19 which is not present in the bucket.
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> fsoDirectoryPathResolver.getAbsolutePathForObjectIDs(
+ Optional.of(Sets.newHashSet(17L, 9L, 10L, 15L, 4L, 3L, 1L, 19L))),
+ "Dir object Ids required but not found in bucket: [19]");
+ }
+
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
index 355cce7774..fa7d99e5a2 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.ozone.om.snapshot;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.lang3.RandomStringUtils;
@@ -48,8 +48,8 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
-import org.apache.hadoop.ozone.om.helpers.WithObjectID;
import
org.apache.hadoop.ozone.om.snapshot.SnapshotDiffObject.SnapshotDiffObjectBuilder;
+import org.apache.hadoop.ozone.om.helpers.WithParentObjectId;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus;
@@ -303,24 +303,26 @@ public class TestSnapshotDiffManager {
}
}
- private Table<String, ? extends WithObjectID> getMockedTable(
- Map<String, WithObjectID> map, String tableName)
+ private Table<String, ? extends WithParentObjectId> getMockedTable(
+ Map<String, WithParentObjectId> map, String tableName)
throws IOException {
- Table<String, ? extends WithObjectID> mocked = mock(Table.class);
+ Table<String, ? extends WithParentObjectId> mocked = mock(Table.class);
Mockito.when(mocked.get(Matchers.any()))
.thenAnswer(invocation -> map.get(invocation.getArgument(0)));
Mockito.when(mocked.getName()).thenReturn(tableName);
return mocked;
}
- private WithObjectID getObjectID(int objectId, int updateId,
- String snapshotTableName) {
+ private WithParentObjectId getKeyInfo(int objectId, int updateId,
+ int parentObjectId,
+ String snapshotTableName) {
String name = "key" + objectId;
if (snapshotTableName.equals(OmMetadataManagerImpl.DIRECTORY_TABLE)) {
return OmDirectoryInfo.newBuilder()
.setObjectID(objectId).setName(name).build();
}
return new OmKeyInfo.Builder().setObjectID(objectId)
+ .setParentObjectID(parentObjectId)
.setVolumeName("vol").setBucketName("bucket").setUpdateID(updateId)
.setReplicationConfig(new ECReplicationConfig(3, 2))
.setKeyName(name).build();
@@ -355,10 +357,10 @@ public class TestSnapshotDiffManager {
throws NativeLibraryNotLoadedException, IOException, RocksDBException {
// Mocking SST file with keys in SST file including tombstones
Set<String> keysWithTombstones = IntStream.range(0, 100)
- .boxed().map(i -> "key" + i).collect(Collectors.toSet());
+ .boxed().map(i -> (i + 100) + "/key" + i).collect(Collectors.toSet());
// Mocking SST file with keys in SST file excluding tombstones
Set<String> keys = IntStream.range(0, 50).boxed()
- .map(i -> "key" + i).collect(Collectors.toSet());
+ .map(i -> (i + 100) + "/key" + i).collect(Collectors.toSet());
// Mocking SSTFileReader functions to return the above keys list.
try (MockedConstruction<ManagedSstFileReader> mockedSSTFileReader =
Mockito.mockConstruction(ManagedSstFileReader.class,
@@ -374,22 +376,23 @@ public class TestSnapshotDiffManager {
})
) {
//
- Map<String, WithObjectID> toSnapshotTableMap =
+ Map<String, WithParentObjectId> toSnapshotTableMap =
IntStream.concat(IntStream.range(0, 25), IntStream.range(50, 100))
- .boxed().collect(Collectors.toMap(i -> "key" + i,
- i -> getObjectID(i, i, snapshotTableName)));
+ .boxed().collect(Collectors.toMap(i -> (i + 100) + "/key" + i,
+ i -> getKeyInfo(i, i, i + 100,
+ snapshotTableName)));
// Mocking To snapshot table containing list of keys b/w 0-25, 50-100
- Table<String, ? extends WithObjectID> toSnapshotTable =
+ Table<String, ? extends WithParentObjectId> toSnapshotTable =
getMockedTable(toSnapshotTableMap, snapshotTableName);
// Mocking To snapshot table containing list of keys b/w 0-50
- Map<String, WithObjectID> fromSnapshotTableMap =
+ Map<String, WithParentObjectId> fromSnapshotTableMap =
IntStream.range(0, 50)
- .boxed().collect(Collectors.toMap(i -> "key" + i,
- i -> getObjectID(i, i, snapshotTableName)));
+ .boxed().collect(Collectors.toMap(i -> (i + 100) + "/key" + i,
+ i -> getKeyInfo(i, i, i + 100, snapshotTableName)));
// Expected Diff 25-50 are newly created keys & keys b/w are deleted,
// when reding keys with tombstones the keys would be added to
// objectIdsToBeChecked otherwise it wouldn't be added
- Table<String, ? extends WithObjectID> fromSnapshotTable =
+ Table<String, ? extends WithParentObjectId> fromSnapshotTable =
getMockedTable(fromSnapshotTableMap, snapshotTableName);
SnapshotDiffManager snapshotDiffManager =
getMockedSnapshotDiffManager(10);
@@ -397,7 +400,7 @@ public class TestSnapshotDiffManager {
// Odd keys should be filtered out in the diff.
Mockito.doAnswer((Answer<Boolean>) invocationOnMock ->
Integer.parseInt(invocationOnMock.getArgument(0, String.class)
- .substring(3)) % 2 == 0).when(snapshotDiffManager)
+ .substring(7)) % 2 == 0).when(snapshotDiffManager)
.isKeyInBucket(Matchers.anyString(), Matchers.anyMap(),
Matchers.anyString());
PersistentMap<byte[], byte[]> oldObjectIdKeyMap =
@@ -406,10 +409,16 @@ public class TestSnapshotDiffManager {
new SnapshotTestUtils.StubbedPersistentMap<>();
PersistentMap<byte[], SnapshotDiffObject> objectIdsToCheck =
new SnapshotTestUtils.StubbedPersistentMap<>();
+ Set<Long> oldParentIds = Sets.newHashSet();
+ Set<Long> newParentIds = Sets.newHashSet();
snapshotDiffManager.addToObjectIdMap(toSnapshotTable,
fromSnapshotTable, Sets.newHashSet("dummy.sst"),
nativeLibraryLoaded, oldObjectIdKeyMap, newObjectIdKeyMap,
- objectIdsToCheck, Maps.newHashMap());
+ objectIdsToCheck, Optional.ofNullable(oldParentIds),
+ Optional.ofNullable(newParentIds),
+ ImmutableMap.of(OmMetadataManagerImpl.DIRECTORY_TABLE, "",
+ OmMetadataManagerImpl.KEY_TABLE, "",
+ OmMetadataManagerImpl.FILE_TABLE, ""));
Iterator<Entry<byte[], byte[]>> oldObjectIdIter =
oldObjectIdKeyMap.iterator();
@@ -574,7 +583,8 @@ public class TestSnapshotDiffManager {
snapshotDiffManager.generateDiffReport("jobId", fromSnapTable,
toSnapTable, objectIdToDiffObject, oldObjectIdKeyMap,
- newObjectIdKeyMap, volumeName, bucketName, fromSnapName, toSnapName);
+ newObjectIdKeyMap, volumeName, bucketName, fromSnapName, toSnapName,
+ false, null, null);
snapshotDiffJob.setStatus(JobStatus.DONE);
snapshotDiffManager.getSnapDiffJobTable().put(jobKey, snapshotDiffJob);
@@ -595,7 +605,7 @@ public class TestSnapshotDiffManager {
actualOrder.add(entry.getType());
long objectId = Long.parseLong(
- DFSUtilClient.bytes2String(entry.getSourcePath()).substring(3));
+ DFSUtilClient.bytes2String(entry.getSourcePath()).substring(4));
Assertions.assertEquals(diffMap.get(objectId), entry.getType());
}
Assertions.assertEquals(expectedOrder, actualOrder);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
