This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new fa9842682a HDDS-9967. Intermittent failure in
TestOzoneRpcClientAbstract.testListSnapshot. (#5970)
fa9842682a is described below
commit fa9842682a49863809b3bd4b05e238c8a91df6d1
Author: Devesh Kumar Singh <[email protected]>
AuthorDate: Thu Jan 18 03:17:17 2024 +0530
HDDS-9967. Intermittent failure in
TestOzoneRpcClientAbstract.testListSnapshot. (#5970)
---
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 4 ---
.../org/apache/hadoop/ozone/om/ListIterator.java | 38 +++++++++++++---------
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 4 +--
3 files changed, 25 insertions(+), 21 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 37b1c129af..da2d7728ba 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -1462,10 +1462,6 @@ public class KeyManagerImpl implements KeyManager {
&& omKeyInfoCacheValue.getCacheValue() == null;
}
- public static boolean isKeyInCache(String key, Table keyTable) {
- return keyTable.getCacheValue(new CacheKey(key)) != null;
- }
-
/**
* Helper function for listStatus to find key in TableCache.
*/
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ListIterator.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ListIterator.java
index 0b2639e16b..7981222c4c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ListIterator.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ListIterator.java
@@ -37,6 +37,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.PriorityQueue;
import java.util.TreeMap;
+import java.util.function.Predicate;
import static
org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
@@ -117,14 +118,17 @@ public class ListIterator {
? extends Table.KeyValue<String, Value>> tableIterator;
private final Table<String, Value> table;
- private HeapEntry currentKey;
+ private HeapEntry currentEntry;
+ private Predicate<String> doesKeyExistInCache;
DbTableIter(int entryIteratorId, Table<String, Value> table,
- String prefixKey, String startKey) throws IOException {
+ String prefixKey, String startKey,
+ Predicate<String> doesKeyExistInCache) throws IOException {
this.entryIteratorId = entryIteratorId;
this.table = table;
this.tableIterator = table.iterator(prefixKey);
- this.currentKey = null;
+ this.currentEntry = null;
+ this.doesKeyExistInCache = doesKeyExistInCache;
// only seek for the start key if the start key is lexicographically
// after the prefix key. For example
@@ -141,11 +145,11 @@ public class ListIterator {
}
private void getNextKey() throws IOException {
- while (tableIterator.hasNext() && currentKey == null) {
+ while (tableIterator.hasNext() && currentEntry == null) {
Table.KeyValue<String, Value> entry = tableIterator.next();
String entryKey = entry.getKey();
- if (!KeyManagerImpl.isKeyInCache(entryKey, table)) {
- currentKey = new HeapEntry(entryIteratorId,
+ if (!doesKeyExistInCache.test(entryKey)) {
+ currentEntry = new HeapEntry(entryIteratorId,
table.getName(), entryKey, entry.getValue());
}
}
@@ -157,13 +161,13 @@ public class ListIterator {
} catch (IOException t) {
throw new UncheckedIOException(t);
}
- return currentKey != null;
+ return currentEntry != null;
}
public HeapEntry next() {
if (hasNext()) {
- HeapEntry ret = currentKey;
- currentKey = null;
+ HeapEntry ret = currentEntry;
+ currentEntry = null;
return ret;
}
throw new NoSuchElementException();
@@ -186,7 +190,6 @@ public class ListIterator {
private final String prefixKey;
private final String startKey;
private final String tableName;
-
private final int entryIteratorId;
CacheIter(int entryIteratorId, String tableName,
@@ -194,7 +197,6 @@ public class ListIterator {
CacheValue<Value>>> cacheIter, String startKey,
String prefixKey) {
this.cacheKeyMap = new TreeMap<>();
-
this.startKey = startKey;
this.prefixKey = prefixKey;
this.tableName = tableName;
@@ -202,7 +204,7 @@ public class ListIterator {
populateCacheMap(cacheIter);
- cacheCreatedKeyIter = cacheKeyMap.entrySet().iterator();
+ cacheCreatedKeyIter = cacheKeyMap.entrySet().stream().filter(e ->
e.getValue() != null).iterator();
}
private void populateCacheMap(Iterator<Map.Entry<CacheKey<String>,
@@ -236,6 +238,10 @@ public class ListIterator {
}
}
+ public boolean doesKeyExistInCache(String key) {
+ return cacheKeyMap.containsKey(key);
+ }
+
public boolean hasNext() {
return cacheCreatedKeyIter.hasNext();
}
@@ -292,11 +298,13 @@ public class ListIterator {
try {
int iteratorId = 0;
for (Table table : tables) {
- iterators.add(new CacheIter<>(iteratorId, table.getName(),
- table.cacheIterator(), startKey, prefixKey));
+ CacheIter cacheIter = new CacheIter<>(iteratorId, table.getName(),
+ table.cacheIterator(), startKey, prefixKey);
+ Predicate<String> doesKeyExistInCache =
cacheIter::doesKeyExistInCache;
+ iterators.add(cacheIter);
iteratorId++;
iterators.add(new DbTableIter<>(iteratorId, table, prefixKey,
- startKey));
+ startKey, doesKeyExistInCache));
iteratorId++;
}
} finally {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index e272cb8692..0f29857723 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -1372,8 +1372,8 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
bucketName, snapshotInfoTable)) {
try {
while (snapshotIterator.hasNext() && maxListResult > 0) {
- SnapshotInfo snapshotInfo = (SnapshotInfo) snapshotIterator.next()
- .getValue();
+ SnapshotInfo snapshotInfo =
+ (SnapshotInfo) snapshotIterator.next().getValue();
if (!snapshotInfo.getName().equals(prevSnapshot)) {
snapshotInfos.add(snapshotInfo);
maxListResult--;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
