This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1bfd93b12d HDDS-8725. Support keyMayExist with ByteBuffer. (#4793)
1bfd93b12d is described below
commit 1bfd93b12d768ed115b8f9aa25a91f0e4d5c1277
Author: Tsz-Wo Nicholas Sze <[email protected]>
AuthorDate: Thu Jun 1 03:16:11 2023 +0800
HDDS-8725. Support keyMayExist with ByteBuffer. (#4793)
---
.../apache/hadoop/hdds/utils/db/CodecBuffer.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 21 +++++++++++
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 21 +++++++++++
.../apache/hadoop/hdds/utils/db/TypedTable.java | 44 ++++++++++++++++++++--
pom.xml | 1 +
5 files changed, 85 insertions(+), 4 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java
index 38d2db8af7..f73cd389fd 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java
@@ -251,7 +251,7 @@ public final class CodecBuffer implements AutoCloseable {
final Integer size = source.apply(buffer);
if (size != null) {
Preconditions.assertTrue(size >= 0, () -> "size = " + size + " < 0");
- if (size <= writable) {
+ if (size > 0 && size <= writable) {
buf.setIndex(buf.readerIndex(), i + size);
}
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java
index 63d0e431c9..c597ad73e6 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java
@@ -167,6 +167,27 @@ class RDBTable implements Table<byte[], byte[]> {
return val;
}
+ Integer getIfExist(ByteBuffer key, ByteBuffer outValue) throws IOException {
+ rdbMetrics.incNumDBKeyGetIfExistChecks();
+ final Supplier<Integer> value = db.keyMayExist(
+ family, key, outValue.duplicate());
+ if (value == null) {
+ return null; // definitely not exists
+ }
+ if (value.get() != null) {
+ // definitely exists, return value size.
+ return value.get();
+ }
+
+ // inconclusive: the key may or may not exist
+ rdbMetrics.incNumDBKeyGetIfExistGets();
+ final Integer val = get(key, outValue);
+ if (val == null) {
+ rdbMetrics.incNumDBKeyGetIfExistMisses();
+ }
+ return val;
+ }
+
@Override
public void delete(byte[] key) throws IOException {
db.delete(family, key);
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
index c800cce171..2e59d7f16c 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java
@@ -37,6 +37,7 @@ import
org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.Holder;
+import org.rocksdb.KeyMayExist;
import org.rocksdb.LiveFileMetaData;
import org.rocksdb.RocksDBException;
import org.slf4j.Logger;
@@ -671,6 +672,26 @@ public final class RocksDatabase implements Closeable {
}
}
+ Supplier<Integer> keyMayExist(ColumnFamily family,
+ ByteBuffer key, ByteBuffer out) throws IOException {
+ assertClose();
+ try {
+ counter.incrementAndGet();
+ final KeyMayExist result = db.get().keyMayExist(
+ family.getHandle(), key, out);
+ switch (result.exists) {
+ case kNotExist: return null;
+ case kExistsWithValue: return () -> result.valueLength;
+ case kExistsWithoutValue: return () -> null;
+ default:
+ throw new IllegalStateException(
+ "Unexpected KeyMayExistEnum case " + result.exists);
+ }
+ } finally {
+ counter.decrementAndGet();
+ }
+ }
+
public ColumnFamily getColumnFamily(String key) {
return columnFamilies.get(key);
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java
index 26e1f5c58c..068722b8ad 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java
@@ -187,6 +187,14 @@ public class TypedTable<KEY, VALUE> implements Table<KEY,
VALUE> {
return true;
} else if (cacheResult.getCacheStatus() == NOT_EXIST) {
return false;
+ } else if (keyCodec.supportCodecBuffer()) {
+ // keyCodec.supportCodecBuffer() is enough since value is not needed.
+ try (CodecBuffer inKey = keyCodec.toDirectCodecBuffer(key)) {
+ // Use zero capacity buffer since value is not needed.
+ try (CodecBuffer outValue = CodecBuffer.allocateDirect(0)) {
+ return getFromTableIfExist(inKey, outValue) != null;
+ }
+ }
} else {
return rawTable.isExist(encodeKey(key));
}
@@ -334,10 +342,40 @@ public class TypedTable<KEY, VALUE> implements Table<KEY,
VALUE> {
}
}
+ private Integer getFromTableIfExist(CodecBuffer key,
+ CodecBuffer outValue) throws IOException {
+ return outValue.putFromSource(
+ buffer -> rawTable.getIfExist(key.asReadOnlyByteBuffer(), buffer));
+ }
+
+ private VALUE getFromTableIfExistCodecBuffer(KEY key) throws IOException {
+ try (CodecBuffer inKey = keyCodec.toDirectCodecBuffer(key)) {
+ for (; ;) {
+ final int allocated = bufferSize.get();
+ try (CodecBuffer outValue = CodecBuffer.allocateDirect(allocated)) {
+ final Integer required = getFromTableIfExist(inKey, outValue);
+ if (required == null) {
+ // key not found
+ return null;
+ } else if (required >= 0 && required <= allocated) {
+ // buffer size is big enough
+ return valueCodec.fromCodecBuffer(outValue);
+ }
+ // buffer size too small, retry
+ increaseBufferSize(required);
+ }
+ }
+ }
+ }
+
private VALUE getFromTableIfExist(KEY key) throws IOException {
- final byte[] keyBytes = encodeKey(key);
- byte[] valueBytes = rawTable.getIfExist(keyBytes);
- return decodeValue(valueBytes);
+ if (supportCodecBuffer) {
+ return getFromTableIfExistCodecBuffer(key);
+ } else {
+ final byte[] keyBytes = encodeKey(key);
+ final byte[] valueBytes = rawTable.getIfExist(keyBytes);
+ return decodeValue(valueBytes);
+ }
}
@Override
diff --git a/pom.xml b/pom.xml
index f7e1a4f59b..2b83f05a46 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1765,6 +1765,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<!-- Allow non-RocksObject classes. -->
<allowedImport>org.rocksdb.ColumnFamilyDescriptor</allowedImport>
<allowedImport>org.rocksdb.CompactionStyle</allowedImport>
+ <allowedImport>org.rocksdb.KeyMayExist</allowedImport>
<allowedImport>org.rocksdb.HistogramData</allowedImport>
<allowedImport>org.rocksdb.HistogramType</allowedImport>
<allowedImport>org.rocksdb.Holder</allowedImport>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
