This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5582b49bd2 HDDS-9454. TypedTable prefix iterator may leak CodecBuffer.
(#5442)
5582b49bd2 is described below
commit 5582b49bd25d02f048571a0930305ecde96f1502
Author: Tsz-Wo Nicholas Sze <[email protected]>
AuthorDate: Sun Oct 15 06:34:19 2023 -0700
HDDS-9454. TypedTable prefix iterator may leak CodecBuffer. (#5442)
---
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 2 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 7 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 78 ++++++++++++++++++++++
3 files changed, 85 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index 371e6abf5e..771b18760a 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@ -287,7 +287,7 @@ public class RDBStore implements DBStore {
}
@Override
- public <K, V> Table<K, V> getTable(String name,
+ public <K, V> TypedTable<K, V> getTable(String name,
Class<K> keyType, Class<V> valueType) throws IOException {
return new TypedTable<>(getTable(name), codecRegistry, keyType,
valueType);
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java
index af54e5b501..e55d841538 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java
@@ -414,7 +414,12 @@ public class TypedTable<KEY, VALUE> implements Table<KEY,
VALUE> {
throws IOException {
if (supportCodecBuffer) {
final CodecBuffer prefixBuffer = encodeKeyCodecBuffer(prefix);
- return newCodecBufferTableIterator(rawTable.iterator(prefixBuffer));
+ try {
+ return newCodecBufferTableIterator(rawTable.iterator(prefixBuffer));
+ } catch (Throwable t) {
+ prefixBuffer.release();
+ throw t;
+ }
} else {
final byte[] prefixBytes = encodeKey(prefix);
return new TypedTableIterator(rawTable.iterator(prefixBytes));
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTableStore.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTableStore.java
index 62366a9520..5b407ed95c 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTableStore.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTableStore.java
@@ -46,11 +46,16 @@ import org.junit.Rule;
import org.rocksdb.RocksDB;
import org.rocksdb.Statistics;
import org.rocksdb.StatsLevel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Tests for RocksDBTable Store.
*/
public class TestRDBTableStore {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestRDBTableStore.class);
+
public static final int MAX_DB_UPDATES_SIZE_THRESHOLD = 80;
private static int count = 0;
private final List<String> families =
@@ -75,6 +80,7 @@ public class TestRDBTableStore {
@BeforeAll
public static void initConstants() {
+ CodecBuffer.enableLeakDetection();
bytesOf = new byte[4][];
for (int i = 1; i <= 3; i++) {
bytesOf[i] = Integer.toString(i).getBytes(StandardCharsets.UTF_8);
@@ -123,6 +129,10 @@ public class TestRDBTableStore {
if (rdbStore != null) {
rdbStore.close();
}
+ if (options != null) {
+ options.close();
+ }
+ CodecTestUtil.gc();
}
@Test
@@ -526,6 +536,7 @@ public class TestRDBTableStore {
}
}
+
@Test
public void testPrefixedIterator() throws Exception {
int containerCount = 3;
@@ -565,6 +576,63 @@ public class TestRDBTableStore {
}
}
+ @Test
+ public void testStringPrefixedIterator() throws Exception {
+ final int prefixCount = 3;
+ final int keyCount = 5;
+ final List<String> prefixes = generatePrefixes(prefixCount);
+ final List<Map<String, String>> data = generateKVs(prefixes, keyCount);
+
+ try (TypedTable<String, String> table = rdbStore.getTable(
+ "PrefixFirst", String.class, String.class)) {
+ populateTable(table, data);
+ for (String prefix : prefixes) {
+ assertIterator(keyCount, prefix, table);
+ }
+
+ final String nonExistingPrefix = RandomStringUtils.random(
+ PREFIX_LENGTH + 2, false, false);
+ assertIterator(0, nonExistingPrefix, table);
+ }
+ }
+
+ static void assertIterator(int expectedCount, String prefix,
+ TypedTable<String, String> table) throws Exception {
+ try (Table.KeyValueIterator<String, String> i = table.iterator(prefix)) {
+ int keyCount = 0;
+ for (; i.hasNext(); keyCount++) {
+ Assertions.assertEquals(prefix,
+ i.next().getKey().substring(0, PREFIX_LENGTH));
+ }
+ Assertions.assertEquals(expectedCount, keyCount);
+
+ // test seekToFirst
+ i.seekToFirst();
+ if (expectedCount > 0) {
+ // iterator should be able to seekToFirst
+ Assertions.assertTrue(i.hasNext());
+ Assertions.assertEquals(prefix,
+ i.next().getKey().substring(0, PREFIX_LENGTH));
+ }
+ }
+ }
+
+ @Test
+ public void testStringPrefixedIteratorCloseDb() throws Exception {
+ try (Table<String, String> testTable = rdbStore.getTable(
+ "PrefixFirst", String.class, String.class)) {
+ // iterator should seek to right pos in the middle
+ rdbStore.close();
+ try {
+ testTable.iterator("abc");
+ Assertions.fail();
+ } catch (IOException ioe) {
+ LOG.info("Great!", ioe);
+ }
+ }
+ }
+
+
@Test
public void testPrefixedRangeKVs() throws Exception {
int containerCount = 3;
@@ -731,4 +799,14 @@ public class TestRDBTableStore {
}
}
}
+
+ private void populateTable(Table<String, String> table,
+ List<Map<String, String>> testData) throws IOException {
+ for (Map<String, String> segment : testData) {
+ for (Map.Entry<String, String> entry : segment.entrySet()) {
+ table.put(entry.getKey(), entry.getValue());
+ LOG.info("put {}", entry);
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
