This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2348784bec HDDS-10288. Checksum to support direct buffers (#6162)
2348784bec is described below
commit 2348784bec6baa118057e78a9716c6a32819a9a9
Author: Duong Nguyen <[email protected]>
AuthorDate: Thu Feb 15 09:14:38 2024 -0800
HDDS-10288. Checksum to support direct buffers (#6162)
---
.../ozone/common/ChecksumByteBufferImpl.java | 29 ++++++++++++++++++++++
.../ozone/common/TestChecksumByteBuffer.java | 19 ++++++++++++++
2 files changed, 48 insertions(+)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java
index 5fab7eacdf..1d596bf700 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferImpl.java
@@ -17,9 +17,13 @@
*/
package org.apache.hadoop.ozone.common;
+import org.apache.hadoop.hdds.JavaUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.util.zip.Checksum;
@@ -35,6 +39,8 @@ public class ChecksumByteBufferImpl implements
ChecksumByteBuffer {
private final Checksum checksum;
private static final Field IS_READY_ONLY_FIELD;
+ // To access Checksum.update(ByteBuffer) API from Java 9+.
+ private static final MethodHandle BYTE_BUFFER_UPDATE;
static {
Field f = null;
@@ -46,6 +52,18 @@ public class ChecksumByteBufferImpl implements
ChecksumByteBuffer {
LOG.error("No isReadOnly field in ByteBuffer", e);
}
IS_READY_ONLY_FIELD = f;
+
+ MethodHandle byteBufferUpdate = null;
+ if (JavaUtils.isJavaVersionAtLeast(9)) {
+ try {
+ byteBufferUpdate =
MethodHandles.publicLookup().findVirtual(Checksum.class, "update",
+ MethodType.methodType(void.class, ByteBuffer.class));
+ } catch (Throwable t) {
+ throw new IllegalStateException("Failed to lookup
Checksum.update(ByteBuffer).");
+ }
+ }
+ BYTE_BUFFER_UPDATE = byteBufferUpdate;
+
}
public ChecksumByteBufferImpl(Checksum impl) {
@@ -57,6 +75,17 @@ public class ChecksumByteBufferImpl implements
ChecksumByteBuffer {
// should be refactored to simply call checksum.update(buffer), as the
// Checksum interface has been enhanced to allow this since Java 9.
public void update(ByteBuffer buffer) {
+ // Prefer JDK9+ implementation that allows ByteBuffer. This allows
DirectByteBuffer to be checksum directly in
+ // native memory.
+ if (BYTE_BUFFER_UPDATE != null) {
+ try {
+ BYTE_BUFFER_UPDATE.invokeExact(checksum, buffer);
+ return;
+ } catch (Throwable e) {
+ throw new IllegalStateException("Error invoking " +
BYTE_BUFFER_UPDATE, e);
+ }
+ }
+
// this is a hack to not do memory copy.
if (IS_READY_ONLY_FIELD != null) {
try {
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java
index 9567fa2c28..0d30d43dc0 100644
---
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.ozone.common;
import org.apache.hadoop.util.PureJavaCrc32;
import org.apache.hadoop.util.PureJavaCrc32C;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import org.apache.commons.lang3.RandomUtils;
import java.util.zip.Checksum;
@@ -45,6 +47,23 @@ public class TestChecksumByteBuffer {
new VerifyChecksumByteBuffer(expected, testee).testCorrectness();
}
+ @Test
+ public void testWithDirectBuffer() {
+ final ChecksumByteBuffer checksum = ChecksumByteBufferFactory.crc32CImpl();
+ byte[] value = "test".getBytes(StandardCharsets.UTF_8);
+ checksum.reset();
+ checksum.update(value, 0, value.length);
+ long checksum1 = checksum.getValue();
+
+ ByteBuffer byteBuffer = ByteBuffer.allocateDirect(value.length);
+ byteBuffer.put(value).rewind();
+ checksum.reset();
+ checksum.update(byteBuffer);
+ long checksum2 = checksum.getValue();
+
+ Assertions.assertEquals(checksum1, checksum2);
+ }
+
static class VerifyChecksumByteBuffer {
private final Checksum expected;
private final ChecksumByteBuffer testee;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
