This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7e8460913b HDDS-8970. Snapshot Diff should return path relative to
bucket root (#5015)
7e8460913b is described below
commit 7e8460913b488dc5923c4907d405dd7531cac72b
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Wed Jul 5 17:40:35 2023 -0700
HDDS-8970. Snapshot Diff should return path relative to bucket root (#5015)
---
.../ozone/snapshot/SnapshotDiffReportOzone.java | 48 +---------------------
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 4 +-
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 22 ++++------
.../ozone/om/snapshot/SnapshotDiffManager.java | 29 ++++++-------
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 2 +-
5 files changed, 29 insertions(+), 76 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
index 14f204999f..c04c09b219 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
@@ -24,15 +24,12 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.db.Codec;
import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
import org.apache.hadoop.hdds.utils.db.Proto2Codec;
-import org.apache.hadoop.hdfs.DFSUtilClient;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DiffReportEntryProto;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffReportProto;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
import java.util.List;
import java.util.stream.Collectors;
@@ -166,7 +163,7 @@ public class SnapshotDiffReportOzone
return null;
}
DiffType type = fromProtobufDiffType(entry.getDiffType());
- return type == null ? null : new DiffReportEntryOzone(type,
+ return type == null ? null : new DiffReportEntry(type,
entry.getSourcePath().getBytes(StandardCharsets.UTF_8),
entry.hasTargetPath() ?
entry.getTargetPath().getBytes(StandardCharsets.UTF_8) : null);
@@ -194,7 +191,7 @@ public class SnapshotDiffReportOzone
public static DiffReportEntry getDiffReportEntry(final DiffType type,
final String sourcePath, final String targetPath) {
- return new DiffReportEntryOzone(type,
+ return new DiffReportEntry(type,
sourcePath.getBytes(StandardCharsets.UTF_8),
targetPath != null ? targetPath.getBytes(StandardCharsets.UTF_8) :
null);
@@ -208,45 +205,4 @@ public class SnapshotDiffReportOzone
public void aggregate(SnapshotDiffReportOzone diffReport) {
this.getDiffList().addAll(diffReport.getDiffList());
}
-
- /**
- * DiffReportEntry for ozone.
- */
- public static class DiffReportEntryOzone extends DiffReportEntry {
-
- public DiffReportEntryOzone(DiffType type, byte[] sourcePath) {
- super(type, sourcePath);
- }
-
- public DiffReportEntryOzone(DiffType type, byte[][] sourcePathComponents) {
- super(type, sourcePathComponents);
- }
-
- public DiffReportEntryOzone(DiffType type, byte[] sourcePath,
- byte[] targetPath) {
- super(type, sourcePath, targetPath);
- }
-
- public DiffReportEntryOzone(DiffType type, byte[][] sourcePathComponents,
- byte[][] targetPathComponents) {
- super(type, sourcePathComponents, targetPathComponents);
- }
-
- static String getPathString(byte[] path) {
- String pathStr = DFSUtilClient.bytes2String(path);
- return pathStr.isEmpty() ? "." : Paths.get(pathStr)
- .toAbsolutePath().toString();
- }
-
- @Override
- public String toString() {
- String str = this.getType().getLabel() + "\t" +
- getPathString(this.getSourcePath());
- if (this.getType() == SnapshotDiffReport.DiffType.RENAME) {
- str = str + " -> " + getPathString(this.getTargetPath());
- }
-
- return str;
- }
- }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index 40b5433d11..c9b0e55b61 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -2484,10 +2484,10 @@ public class TestRootedOzoneFileSystem {
Assert.assertEquals(SnapshotDiffReport.DiffType.CREATE,
diff.getDiffList().get(1).getType());
Assert.assertArrayEquals(
- "/key1".getBytes(StandardCharsets.UTF_8),
+ "key1".getBytes(StandardCharsets.UTF_8),
diff.getDiffList().get(0).getSourcePath());
Assert.assertArrayEquals(
- "/key2".getBytes(StandardCharsets.UTF_8),
+ "key2".getBytes(StandardCharsets.UTF_8),
diff.getDiffList().get(1).getSourcePath());
// test whether snapdiff returns aggregated response as
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index f64594f76f..dd58791c48 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -88,7 +88,6 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static
org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isDone;
import static
org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isStarting;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
@@ -616,7 +615,7 @@ public class TestOmSnapshot {
snap1, snap2);
Assertions.assertEquals(Arrays.asList(
SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReport.DiffType.RENAME, "/dir1", "/dir1_rename")),
+ SnapshotDiffReport.DiffType.RENAME, "dir1", "dir1_rename")),
diff.getDiffList());
}
@@ -659,11 +658,9 @@ public class TestOmSnapshot {
Assert.assertEquals(2, diff2.getDiffList().size());
Assert.assertEquals(
Arrays.asList(SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReport.DiffType.DELETE,
- OZONE_URI_DELIMITER + key1),
+ SnapshotDiffReport.DiffType.DELETE, key1),
SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReport.DiffType.CREATE,
- OZONE_URI_DELIMITER + key2)),
+ SnapshotDiffReport.DiffType.CREATE, key2)),
diff2.getDiffList());
// Rename Key2
@@ -677,8 +674,8 @@ public class TestOmSnapshot {
Assert.assertEquals(1, diff3.getDiffList().size());
Assert.assertTrue(diff3.getDiffList().contains(
SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReportOzone.DiffType.RENAME, OZONE_URI_DELIMITER +
key2,
- OZONE_URI_DELIMITER + key2Renamed)));
+ SnapshotDiffReportOzone.DiffType.RENAME, key2,
+ key2Renamed)));
// Create a directory
@@ -691,8 +688,7 @@ public class TestOmSnapshot {
Assert.assertEquals(1, diff4.getDiffList().size());
Assert.assertTrue(diff4.getDiffList().contains(
SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReportOzone.DiffType.CREATE,
- OM_KEY_PREFIX + dir1)));
+ SnapshotDiffReportOzone.DiffType.CREATE, dir1)));
String key3 = createFileKeyWithPrefix(bucket1, "key-3-");
String snap6 = "snap" + counter.incrementAndGet();
@@ -707,10 +703,10 @@ public class TestOmSnapshot {
diff5 = getSnapDiffReport(volume, bucket, snap6, snap7);
List<SnapshotDiffReport.DiffReportEntry> expectedDiffList =
Arrays.asList(SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReport.DiffType.RENAME, OZONE_URI_DELIMITER + key3,
- OZONE_URI_DELIMITER + renamedKey3),
+ SnapshotDiffReport.DiffType.RENAME, key3,
+ renamedKey3),
SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReport.DiffType.MODIFY, OZONE_URI_DELIMITER + key3)
+ SnapshotDiffReport.DiffType.MODIFY, key3)
);
assertEquals(expectedDiffList, diff5.getDiffList());
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
index 0378e9edc0..19f2976978 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
@@ -1266,8 +1266,8 @@ public class SnapshotDiffManager implements AutoCloseable
{
}
@SuppressFBWarnings("DMI_HARDCODED_ABSOLUTE_FILENAME")
- private String resolveAbsolutePath(boolean isFSOBucket,
- final Optional<Map<Long, Path>> parentIdMap, byte[] keyVal)
+ private String resolveBucketRelativePath(boolean isFSOBucket,
+ final Optional<Map<Long, Path>> parentIdMap, byte[] keyVal)
throws IOException {
String key = codecRegistry.asObject(keyVal, String.class);
if (isFSOBucket) {
@@ -1279,9 +1279,10 @@ public class SnapshotDiffManager implements
AutoCloseable {
parentId));
}
return parentIdMap.map(m -> m.get(parentId).resolve(splitKey[1]))
- .get().toString();
+ .get().toString().substring(1);
}
- return Paths.get(OzoneConsts.OZONE_URI_DELIMITER).resolve(key).toString();
+ return Paths.get(OzoneConsts.OZONE_URI_DELIMITER).resolve(key).toString()
+ .substring(1);
}
@SuppressWarnings({"checkstyle:ParameterNumber", "checkstyle:MethodLength"})
@@ -1366,28 +1367,28 @@ public class SnapshotDiffManager implements
AutoCloseable {
throw new IllegalStateException(
"Old and new key name both are null");
} else if (oldKeyName == null) { // Key Created.
- String key = resolveAbsolutePath(isFSOBucket, newParentIdPathMap,
- newKeyName);
+ String key = resolveBucketRelativePath(isFSOBucket,
+ newParentIdPathMap, newKeyName);
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(CREATE, key);
createDiffs.add(codecRegistry.asRawData(entry));
} else if (newKeyName == null) { // Key Deleted.
- String key = resolveAbsolutePath(isFSOBucket, oldParentIdPathMap,
- oldKeyName);
+ String key = resolveBucketRelativePath(isFSOBucket,
+ oldParentIdPathMap, oldKeyName);
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(DELETE, key);
deleteDiffs.add(codecRegistry.asRawData(entry));
} else if (Arrays.equals(oldKeyName, newKeyName)) { // Key modified.
- String key = resolveAbsolutePath(isFSOBucket, newParentIdPathMap,
- newKeyName);
+ String key = resolveBucketRelativePath(isFSOBucket,
+ newParentIdPathMap, newKeyName);
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(MODIFY, key);
modifyDiffs.add(codecRegistry.asRawData(entry));
} else { // Key Renamed.
- String oldKey = resolveAbsolutePath(isFSOBucket,
oldParentIdPathMap,
- oldKeyName);
- String newKey = resolveAbsolutePath(isFSOBucket,
newParentIdPathMap,
- newKeyName);
+ String oldKey = resolveBucketRelativePath(isFSOBucket,
+ oldParentIdPathMap, oldKeyName);
+ String newKey = resolveBucketRelativePath(isFSOBucket,
+ newParentIdPathMap, newKeyName);
renameDiffs.add(codecRegistry.asRawData(
SnapshotDiffReportOzone.getDiffReportEntry(RENAME, oldKey,
newKey)));
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
index 8a3f124d28..7371495113 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
@@ -846,7 +846,7 @@ public class TestSnapshotDiffManager {
actualOrder.add(entry.getType());
long objectId = Long.parseLong(
- DFSUtilClient.bytes2String(entry.getSourcePath()).substring(4));
+ DFSUtilClient.bytes2String(entry.getSourcePath()).substring(3));
assertEquals(diffMap.get(objectId), entry.getType());
}
assertEquals(expectedOrder, actualOrder);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
