This is an automated email from the ASF dual-hosted git repository.
jacksonyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5adbef2135 HDDS-6127. file checksum to support both CRC32 and CRC32C.
(#3570)
5adbef2135 is described below
commit 5adbef2135e3f14abfbb7816e2107b51d557a55e
Author: Wei-Chiu Chuang <[email protected]>
AuthorDate: Fri Aug 12 17:09:56 2022 +0800
HDDS-6127. file checksum to support both CRC32 and CRC32C. (#3570)
---
.../client/checksum/BaseFileChecksumHelper.java | 40 +++++++++++++++++++---
.../checksum/ReplicatedFileChecksumHelper.java | 4 ++-
.../checksum/TestReplicatedFileChecksumHelper.java | 11 +++++-
3 files changed, 49 insertions(+), 6 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/BaseFileChecksumHelper.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/BaseFileChecksumHelper.java
index 792a9e3616..93d5f71422 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/BaseFileChecksumHelper.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/BaseFileChecksumHelper.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.ozone.client.checksum;
import org.apache.hadoop.fs.CompositeCrcFileChecksum;
import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.MD5Hash;
@@ -57,6 +59,7 @@ public abstract class BaseFileChecksumHelper {
private ClientProtocol rpcClient;
private OzoneClientConfig.ChecksumCombineMode combineMode;
+ private ContainerProtos.ChecksumType checksumType;
private final DataOutputBuffer blockChecksumBuf = new DataOutputBuffer();
private XceiverClientFactory xceiverClientFactory;
@@ -111,6 +114,10 @@ public abstract class BaseFileChecksumHelper {
return combineMode;
}
+ protected ContainerProtos.ChecksumType getChecksumType() {
+ return checksumType;
+ }
+
protected XceiverClientFactory getXceiverClientFactory() {
return xceiverClientFactory;
}
@@ -139,6 +146,10 @@ public abstract class BaseFileChecksumHelper {
this.bytesPerCRC = bytesPerCRC;
}
+ protected void setChecksumType(ContainerProtos.ChecksumType type) {
+ checksumType = type;
+ }
+
/**
* Request the blocks created in the most recent version from Ozone Manager.
*
@@ -242,8 +253,29 @@ public abstract class BaseFileChecksumHelper {
//compute file MD5
final MD5Hash fileMD5 = MD5Hash.digest(getBlockChecksumBuf().getData());
// assume CRC32 for now
- return new MD5MD5CRC32GzipFileChecksum(getBytesPerCRC(),
- crcPerBlock, fileMD5);
+ switch (getChecksumType()) {
+ case CRC32:
+ return new MD5MD5CRC32GzipFileChecksum(getBytesPerCRC(),
+ crcPerBlock, fileMD5);
+ case CRC32C:
+ return new MD5MD5CRC32CastagnoliFileChecksum(getBytesPerCRC(),
+ crcPerBlock, fileMD5);
+ default:
+ throw new IllegalArgumentException("unexpected checksum type "
+ + getChecksumType());
+ }
+
+ }
+
+ DataChecksum.Type toHadoopChecksumType() {
+ switch (checksumType) {
+ case CRC32:
+ return DataChecksum.Type.CRC32;
+ case CRC32C:
+ return DataChecksum.Type.CRC32C;
+ default:
+ throw new IllegalArgumentException("unsupported checksum type");
+ }
}
FileChecksum makeCompositeCrcResult() throws IOException {
@@ -252,7 +284,7 @@ public abstract class BaseFileChecksumHelper {
blockSizeHint = keyLocationInfos.get(0).getLength();
}
CrcComposer crcComposer =
- CrcComposer.newCrcComposer(DataChecksum.Type.CRC32, blockSizeHint);
+ CrcComposer.newCrcComposer(toHadoopChecksumType(), blockSizeHint);
byte[] blockChecksumBytes = blockChecksumBuf.getData();
for (int i = 0; i < keyLocationInfos.size(); ++i) {
@@ -270,7 +302,7 @@ public abstract class BaseFileChecksumHelper {
int compositeCrc = CrcUtil.readInt(crcComposer.digest(), 0);
return new CompositeCrcFileChecksum(
- compositeCrc, DataChecksum.Type.CRC32, bytesPerCRC);
+ compositeCrc, toHadoopChecksumType(), bytesPerCRC);
}
public FileChecksum getFileChecksum() {
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedFileChecksumHelper.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedFileChecksumHelper.java
index 84a44515f0..7c45062b4c 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedFileChecksumHelper.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedFileChecksumHelper.java
@@ -71,7 +71,6 @@ public class ReplicatedFileChecksumHelper extends
BaseFileChecksumHelper {
blockIdx++) {
OmKeyLocationInfo keyLocationInfo =
getKeyLocationInfoList().get(blockIdx);
- currentLength += keyLocationInfo.getLength();
if (currentLength > getLength()) {
return;
}
@@ -81,6 +80,8 @@ public class ReplicatedFileChecksumHelper extends
BaseFileChecksumHelper {
"Fail to get block checksum for " + keyLocationInfo
+ ", checksum combine mode : {}" + getCombineMode());
}
+
+ currentLength += keyLocationInfo.getLength();
}
}
@@ -106,6 +107,7 @@ public class ReplicatedFileChecksumHelper extends
BaseFileChecksumHelper {
ContainerProtos.ChecksumData checksumData =
chunkInfos.get(0).getChecksumData();
+ setChecksumType(checksumData.getType());
int bytesPerChecksum = checksumData.getBytesPerChecksum();
setBytesPerCRC(bytesPerChecksum);
diff --git
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/checksum/TestReplicatedFileChecksumHelper.java
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/checksum/TestReplicatedFileChecksumHelper.java
index 589936bc42..3954bb9ba6 100644
---
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/checksum/TestReplicatedFileChecksumHelper.java
+++
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/checksum/TestReplicatedFileChecksumHelper.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.client.checksum;
import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
@@ -87,6 +88,11 @@ public class TestReplicatedFileChecksumHelper {
@Before
public void init() throws IOException {
ConfigurationSource config = new InMemoryConfiguration();
+ OzoneClientConfig clientConfig = config.getObject(OzoneClientConfig.class);
+ clientConfig.setChecksumType(ContainerProtos.ChecksumType.CRC32C);
+
+ ((InMemoryConfiguration)config).setFromObject(clientConfig);
+
rpcClient = new RpcClient(config, null) {
@Override
@@ -104,6 +110,7 @@ public class TestReplicatedFileChecksumHelper {
return new MockXceiverClientFactory();
}
};
+
client = new OzoneClient(config, rpcClient);
store = client.getObjectStore();
@@ -357,7 +364,9 @@ public class TestReplicatedFileChecksumHelper {
helper.compute();
FileChecksum fileChecksum = helper.getFileChecksum();
- assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
+ //assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
+ assertEquals(DataChecksum.Type.CRC32C,
+ ((MD5MD5CRC32FileChecksum)fileChecksum).getCrcType());
assertEquals(1, helper.getKeyLocationInfoList().size());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
