This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 011de37b19 HDDS-9041. Intermittent Delete root failed. (#5204)
011de37b19 is described below
commit 011de37b19ec874819ae5bbf726580ba5b0b7f93
Author: devmadhuu <[email protected]>
AuthorDate: Thu Aug 24 11:12:10 2023 +0530
HDDS-9041. Intermittent Delete root failed. (#5204)
---
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 24 ++++++++++++----------
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 6 ------
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 23 +++++++++++++--------
.../hadoop/ozone/om/OzoneListStatusHelper.java | 15 +++++++-------
4 files changed, 35 insertions(+), 33 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index c2325b5ae1..17100cdd38 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -227,11 +227,9 @@ public class TestOzoneFileSystem {
@After
public void cleanup() {
try {
- FileStatus[] fileStatuses = fs.listStatus(ROOT);
- for (FileStatus fileStatus : fileStatuses) {
- fs.delete(fileStatus.getPath(), true);
- }
- } catch (IOException ex) {
+ deleteRootDir();
+ } catch (IOException | InterruptedException ex) {
+ LOG.error("Failed to cleanup files.", ex);
fail("Failed to cleanup files.");
}
}
@@ -797,21 +795,25 @@ public class TestOzoneFileSystem {
*
* @throws IOException DB failure
*/
- protected void deleteRootDir() throws IOException {
+ protected void deleteRootDir() throws IOException, InterruptedException {
FileStatus[] fileStatuses = fs.listStatus(ROOT);
if (fileStatuses == null) {
return;
}
+ deleteRootRecursively(fileStatuses);
+ fileStatuses = fs.listStatus(ROOT);
+ if (fileStatuses != null) {
+ Assert.assertEquals(
+ "Delete root failed!", 0, fileStatuses.length);
+ }
+ }
+ private static void deleteRootRecursively(FileStatus[] fileStatuses)
+ throws IOException {
for (FileStatus fStatus : fileStatuses) {
fs.delete(fStatus.getPath(), true);
}
-
- fileStatuses = fs.listStatus(ROOT);
- if (fileStatuses != null) {
- Assert.assertEquals("Delete root failed!", 0, fileStatuses.length);
- }
}
/**
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
index ed148bc45f..ce637f8ea9 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java
@@ -85,12 +85,6 @@ public class TestOzoneFileSystemWithFSO extends
TestOzoneFileSystem {
@Override
public void cleanup() {
super.cleanup();
- try {
- deleteRootDir();
- } catch (IOException e) {
- LOG.info("Failed to cleanup DB tables.", e);
- fail("Failed to cleanup DB tables." + e.getMessage());
- }
}
private static final Logger LOG =
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
index 55ffca602a..2b9fa98736 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java
@@ -70,6 +70,7 @@ import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.client.ReplicationType.RATIS;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
@@ -80,7 +81,8 @@ import static org.junit.Assert.fail;
* Tests to verify Object store with prefix enabled cases.
*/
public class TestObjectStoreWithFSO {
-
+ private static final Path ROOT =
+ new Path(OZONE_URI_DELIMITER);
private static MiniOzoneCluster cluster = null;
private static OzoneConfiguration conf;
private static String clusterId;
@@ -138,22 +140,25 @@ public class TestObjectStoreWithFSO {
*
* @throws IOException DB failure
*/
- private void deleteRootDir() throws IOException {
- Path root = new Path("/");
- FileStatus[] fileStatuses = fs.listStatus(root);
+ protected void deleteRootDir() throws IOException {
+ FileStatus[] fileStatuses = fs.listStatus(ROOT);
if (fileStatuses == null) {
return;
}
+ deleteRootRecursively(fileStatuses);
+ fileStatuses = fs.listStatus(ROOT);
+ if (fileStatuses != null) {
+ Assert.assertEquals(
+ "Delete root failed!", 0, fileStatuses.length);
+ }
+ }
+ private static void deleteRootRecursively(FileStatus[] fileStatuses)
+ throws IOException {
for (FileStatus fStatus : fileStatuses) {
fs.delete(fStatus.getPath(), true);
}
-
- fileStatuses = fs.listStatus(root);
- if (fileStatuses != null) {
- Assert.assertEquals("Delete root failed!", 0, fileStatuses.length);
- }
}
@Test
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java
index 2c13e5cb5d..758206f200 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java
@@ -51,6 +51,7 @@ import java.util.PriorityQueue;
import java.util.NoSuchElementException;
import java.util.Collection;
import java.util.Collections;
+import java.util.stream.Collectors;
import static org.apache.hadoop.ozone.om.exceptions.OMException.
@@ -214,11 +215,12 @@ public class OzoneListStatusHelper {
HeapEntry entry = heapIterator.next();
OzoneFileStatus status = entry.getStatus(prefixKey,
scmBlockSize, volumeName, bucketName, replication);
- map.put(entry.key, status);
+ map.putIfAbsent(entry.key, status);
}
}
- return map.values();
+ return map.values().stream().filter(e -> e != null).collect(
+ Collectors.toList());
}
private String getDbKey(String key, OmKeyArgs args,
@@ -285,7 +287,7 @@ public class OzoneListStatusHelper {
private final Object value;
HeapEntry(EntryType entryType, String key, Object value) {
- Preconditions.checkArgument(
+ Preconditions.checkArgument(value == null ||
value instanceof OmDirectoryInfo ||
value instanceof OmKeyInfo);
this.entryType = entryType;
@@ -322,6 +324,9 @@ public class OzoneListStatusHelper {
String bucketName,
ReplicationConfig bucketReplication
) {
+ if (value == null) {
+ return null;
+ }
OmKeyInfo keyInfo;
if (entryType.isDir()) {
Preconditions.checkArgument(value instanceof OmDirectoryInfo);
@@ -455,10 +460,6 @@ public class OzoneListStatusHelper {
cacheIter.next();
String cacheKey = entry.getKey().getCacheKey();
Value cacheOmInfo = entry.getValue().getCacheValue();
- // cacheOmKeyInfo is null if an entry is deleted in cache
- if (cacheOmInfo == null) {
- continue;
- }
// Copy cache value to local copy and work on it
if (cacheOmInfo instanceof CopyObject) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
