This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c4d9456120 HDDS-8968. [Snapshot] Distcp using snapshots fails due to
incorrect toSnapshot name. (#5040)
c4d9456120 is described below
commit c4d9456120868aedf0635aac6e6d82b92a0bbe37
Author: Sadanand Shenoy <[email protected]>
AuthorDate: Thu Jul 13 10:46:48 2023 +0530
HDDS-8968. [Snapshot] Distcp using snapshots fails due to incorrect
toSnapshot name. (#5040)
---
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 6 ++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 7 ++++-
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 12 --------
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 6 ----
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 33 +++++++++++++++-------
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 33 ++++++++++++++--------
6 files changed, 56 insertions(+), 41 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
index 6d3268a144..c20f611473 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.ozone.om.helpers;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
import javax.annotation.Nonnull;
import java.nio.file.Paths;
+import java.util.UUID;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
@@ -278,4 +280,8 @@ public final class OzoneFSUtils {
return key;
}
}
+
+ public static String generateUniqueTempSnapshotName() {
+ return "temp" + UUID.randomUUID() + SnapshotInfo.generateName(Time.now());
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index a180a8e432..d3107a1f2f 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -2506,9 +2506,14 @@ public class TestRootedOzoneFileSystem {
Path file =
new Path(bucketPath1, "key" + RandomStringUtils.randomAlphabetic(5));
ContractTestUtils.touch(fs, file);
- diff = ofs.getSnapshotDiffReport(bucketPath1, toSnap, ".");
+ diff = ofs.getSnapshotDiffReport(bucketPath1, toSnap, "");
Assert.assertEquals(1, diff.getDiffList().size());
+ diff = ofs.getSnapshotDiffReport(bucketPath1, "", toSnap);
+ Assert.assertEquals(1, diff.getDiffList().size());
+
+ diff = ofs.getSnapshotDiffReport(bucketPath1, "", "");
+ Assert.assertEquals(0, diff.getDiffList().size());
// try snapDiff between non-bucket paths
String errorMsg = "Path is not a bucket";
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index de0f8f1f79..f30ed1a546 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -110,7 +110,6 @@ import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.CONTAINS_SNAPSHOT;
-import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INTERNAL_ERROR;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION;
import static
org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
@@ -1477,17 +1476,6 @@ public class TestOmSnapshot {
null, 0, false, disableNativeDiff));
assertEquals(KEY_NOT_FOUND, omException.getResult());
-
- createSnapshot(volume, bucket, snap2);
-
- omException = assertThrows(OMException.class, () ->
- store.snapshotDiff(volume, bucket, snap2, snap1, null, 0, false,
- disableNativeDiff)
- );
-
- assertEquals(INTERNAL_ERROR, omException.getResult());
- assertEquals("fromSnapshot:" + snap2 + " should be older than to " +
- "toSnapshot:" + snap1, omException.getMessage());
}
@Test
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 7f6b8050a9..7465319888 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -821,12 +821,6 @@ public final class OmSnapshotManager implements
AutoCloseable {
// Block SnapDiff if either of the snapshots is not active.
checkSnapshotActive(fromSnapInfo, false);
checkSnapshotActive(toSnapInfo, false);
-
- // Check snapshot creation time
- if (fromSnapInfo.getCreationTime() > toSnapInfo.getCreationTime()) {
- throw new IOException("fromSnapshot:" + fromSnapInfo.getName() +
- " should be older than to toSnapshot:" + toSnapInfo.getName());
- }
}
private int getIndexFromToken(final String token) throws IOException {
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index 4dc97d3a90..b79a24b356 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -65,7 +65,6 @@ import
org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
-import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
@@ -77,7 +76,6 @@ import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
-import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -668,11 +666,21 @@ public class BasicOzoneClientAdapterImpl implements
OzoneClientAdapter {
public SnapshotDiffReport getSnapshotDiffReport(Path snapshotDir,
String fromSnapshot, String toSnapshot)
throws IOException, InterruptedException {
- boolean takeTemporarySnapshot = false;
- if (toSnapshot.equals(ACTIVE_FS_SNAPSHOT_NAME)) {
- takeTemporarySnapshot = true;
+ boolean takeTemporaryToSnapshot = false;
+ boolean takeTemporaryFromSnapshot = false;
+ if (toSnapshot.isEmpty()) {
+ // empty toSnapshot implies diff b/w the fromSnapshot &
+ // current state.
+ takeTemporaryToSnapshot = true;
toSnapshot = createSnapshot(snapshotDir.toString(),
- "temp" + SnapshotInfo.generateName(Time.now()));
+ OzoneFSUtils.generateUniqueTempSnapshotName());
+ }
+ if (fromSnapshot.isEmpty()) {
+ // empty fromSnapshot implies diff b/w the current state
+ // & the toSnapshot
+ takeTemporaryFromSnapshot = true;
+ fromSnapshot = createSnapshot(snapshotDir.toString(),
+ OzoneFSUtils.generateUniqueTempSnapshotName());
}
try {
SnapshotDiffReportOzone aggregated;
@@ -692,11 +700,16 @@ public class BasicOzoneClientAdapterImpl implements
OzoneClientAdapter {
return aggregated;
} finally {
// delete the temp snapshot
- if (takeTemporarySnapshot) {
+ if (takeTemporaryToSnapshot || takeTemporaryFromSnapshot) {
OFSPath snapPath = new OFSPath(snapshotDir.toString(), config);
- ozoneClient.getObjectStore()
- .deleteSnapshot(snapPath.getVolumeName(), snapPath.getBucketName(),
- toSnapshot);
+ if (takeTemporaryToSnapshot) {
+ objectStore.deleteSnapshot(snapPath.getVolumeName(),
+ snapPath.getBucketName(), toSnapshot);
+ }
+ if (takeTemporaryFromSnapshot) {
+ objectStore.deleteSnapshot(snapPath.getVolumeName(),
+ snapPath.getBucketName(), fromSnapshot);
+ }
}
}
}
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 1ad2dc34fd..1e364b3f77 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -75,7 +75,6 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
-import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
@@ -84,7 +83,6 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,8 +116,6 @@ public class BasicRootedOzoneClientAdapterImpl
private BucketLayout defaultOFSBucketLayout;
private OzoneConfiguration config;
- public static final String ACTIVE_FS_SNAPSHOT_NAME = ".";
-
/**
* Create new OzoneClientAdapter implementation.
*
@@ -1318,11 +1314,21 @@ public class BasicRootedOzoneClientAdapterImpl
public SnapshotDiffReport getSnapshotDiffReport(Path snapshotDir,
String fromSnapshot, String toSnapshot)
throws IOException, InterruptedException {
- boolean takeTemporarySnapshot = false;
- if (toSnapshot.equals(ACTIVE_FS_SNAPSHOT_NAME)) {
- takeTemporarySnapshot = true;
+ boolean takeTemporaryToSnapshot = false;
+ boolean takeTemporaryFromSnapshot = false;
+ if (toSnapshot.isEmpty()) {
+ // empty toSnapshot implies diff b/w the fromSnapshot &
+ // current state.
+ takeTemporaryToSnapshot = true;
toSnapshot = createSnapshot(snapshotDir.toString(),
- "temp" + SnapshotInfo.generateName(Time.now()));
+ OzoneFSUtils.generateUniqueTempSnapshotName());
+ }
+ if (fromSnapshot.isEmpty()) {
+ // empty fromSnapshot implies diff b/w the current state
+ // & the toSnapshot
+ takeTemporaryFromSnapshot = true;
+ fromSnapshot = createSnapshot(snapshotDir.toString(),
+ OzoneFSUtils.generateUniqueTempSnapshotName());
}
OFSPath ofsPath = new OFSPath(snapshotDir, config);
String volume = ofsPath.getVolumeName();
@@ -1347,10 +1353,13 @@ public class BasicRootedOzoneClientAdapterImpl
return aggregated;
} finally {
// delete the temp snapshot
- if (takeTemporarySnapshot) {
- ozoneClient.getObjectStore()
- .deleteSnapshot(ofsPath.getVolumeName(), ofsPath.getBucketName(),
- toSnapshot);
+ if (takeTemporaryToSnapshot) {
+ objectStore.deleteSnapshot(ofsPath.getVolumeName(),
+ ofsPath.getBucketName(), toSnapshot);
+ }
+ if (takeTemporaryFromSnapshot) {
+ objectStore.deleteSnapshot(ofsPath.getVolumeName(),
+ ofsPath.getBucketName(), fromSnapshot);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
