HDDS-885. Fix test failures due to ChecksumData. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef3b03b7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef3b03b7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef3b03b7 Branch: refs/heads/HDFS-12943 Commit: ef3b03b75abd5ca5b96dab7985aa8fe17578fa07 Parents: fb10803 Author: Bharat Viswanadham <bha...@apache.org> Authored: Mon Dec 3 10:52:49 2018 -0800 Committer: Bharat Viswanadham <bha...@apache.org> Committed: Mon Dec 3 10:52:49 2018 -0800 ---------------------------------------------------------------------- .../hdds/scm/storage/ContainerProtocolCalls.java | 14 +++++++++++--- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +++-- .../org/apache/hadoop/ozone/common/Checksum.java | 14 ++++++++++++-- .../ozone/container/common/helpers/ChunkInfo.java | 9 ++++++++- .../common/src/main/resources/ozone-default.xml | 18 ++++++++++++++++++ .../container/common/impl/TestHddsDispatcher.java | 2 ++ .../apache/hadoop/ozone/client/rpc/RpcClient.java | 5 +++-- .../common/TestBlockDeletingService.java | 4 ++-- .../container/common/helpers/TestBlockData.java | 7 ++++++- .../web/storage/DistributedStorageHandler.java | 5 +++-- 10 files changed, 68 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java index 1f20d00..8af3973 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java @@ -21,6 +21,8 @@ package org.apache.hadoop.hdds.scm.storage; import org.apache.hadoop.hdds.scm.XceiverClientAsyncReply; import org.apache.hadoop.hdds.scm.container.common.helpers .BlockNotCommittedException; +import org.apache.hadoop.ozone.common.Checksum; +import org.apache.hadoop.ozone.common.ChecksumData; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.container.common.helpers @@ -305,10 +307,16 @@ public final class ContainerProtocolCalls { KeyValue keyValue = KeyValue.newBuilder().setKey("OverWriteRequested").setValue("true") .build(); + Checksum checksum = new Checksum(); + ChecksumData checksumData = checksum.computeChecksum(data); ChunkInfo chunk = - ChunkInfo.newBuilder().setChunkName(blockID.getLocalID() - + "_chunk").setOffset(0).setLen(data.length). - addMetadata(keyValue).build(); + ChunkInfo.newBuilder() + .setChunkName(blockID.getLocalID() + "_chunk") + .setOffset(0) + .setLen(data.length) + .addMetadata(keyValue) + .setChecksumData(checksumData.getProtoBufMessage()) + .build(); PutSmallFileRequestProto putSmallFileRequest = PutSmallFileRequestProto.newBuilder().setChunkInfo(chunk) http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 496861c..a331f48 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -342,8 +342,9 @@ public final class OzoneConfigKeys { public static final String OZONE_CLIENT_CHECKSUM_TYPE_DEFAULT = "SHA256"; public static final String OZONE_CLIENT_BYTES_PER_CHECKSUM = "ozone.client.bytes.per.checksum"; - public static final int OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT = - 1024 * 1024; // 1 MB + public static final String OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT = "1MB"; + public static final int OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT_BYTES = + 1024 * 1024; public static final int OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE = 256 * 1024; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java index 83293e5..56ad71b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java @@ -24,7 +24,9 @@ import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ChecksumType; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -67,7 +69,7 @@ public class Checksum { this.checksumType = ChecksumType.valueOf( OzoneConfigKeys.OZONE_CLIENT_CHECKSUM_TYPE_DEFAULT); this.bytesPerChecksum = OzoneConfigKeys - .OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT; + .OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT_BYTES; // Default is 1MB } /** @@ -236,4 +238,12 @@ public class Checksum { return checksumData.verifyChecksumDataMatches(computedChecksumData); } + + /** + * Returns a ChecksumData with type NONE for testing. + */ + @VisibleForTesting + public static ContainerProtos.ChecksumData getNoChecksumDataProto() { + return new ChecksumData(ChecksumType.NONE, 0).getProtoBufMessage(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java index d75f10f..1c73a31 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import java.io.IOException; import java.util.Map; import java.util.TreeMap; +import org.apache.hadoop.ozone.common.Checksum; import org.apache.hadoop.ozone.common.ChecksumData; /** @@ -105,7 +106,13 @@ public class ChunkInfo { builder.setChunkName(this.getChunkName()); builder.setOffset(this.getOffset()); builder.setLen(this.getLen()); - builder.setChecksumData(this.checksumData.getProtoBufMessage()); + if (checksumData == null) { + // ChecksumData cannot be null while computing the protobufMessage. + // Set it to NONE type (equivalent to non checksum). + builder.setChecksumData(Checksum.getNoChecksumDataProto()); + } else { + builder.setChecksumData(this.checksumData.getProtoBufMessage()); + } for (Map.Entry<String, String> entry : metadata.entrySet()) { ContainerProtos.KeyValue.Builder keyValBuilder = http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/common/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index edce616..d8e6c78 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1397,5 +1397,23 @@ </description> </property> + <property> + <name>ozone.client.checksum.type</name> + <value>SHA256</value> + <tag>OZONE, CLIENT, MANAGEMENT</tag> + <description>The checksum type [NONE/ CRC32/ CRC32C/ SHA256/ MD5] determines + which algorithm would be used to compute checksum for chunk data. + Default checksum type is SHA256. + </description> + </property> + + <property> + <name>ozone.client.bytes.per.checksum</name> + <value>1MB</value> + <tag>OZONE, CLIENT, MANAGEMENT</tag> + <description>Checksum will be computed for every bytes per checksum number + of bytes and stored sequentially. + </description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java index 933ed70..4727261 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .WriteChunkRequestProto; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerAction; +import org.apache.hadoop.ozone.common.Checksum; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.Handler; @@ -199,6 +200,7 @@ public class TestHddsDispatcher { + containerId + "_chunk_" + localId) .setOffset(0) .setLen(data.size()) + .setChecksumData(Checksum.getNoChecksumDataProto()) .build(); WriteChunkRequestProto.Builder writeChunkRequest = WriteChunkRequestProto http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 65adbfa..334f713 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -171,9 +171,10 @@ public class RpcClient implements ClientProtocol { OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS); - int configuredChecksumSize = conf.getInt( + int configuredChecksumSize = (int) conf.getStorageSize( OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM, - OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT); + OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT, + StorageUnit.BYTES); int checksumSize; if(configuredChecksumSize < OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index 0db047f..e8ae266 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.common.Checksum; import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.impl.ContainerData; @@ -140,8 +141,7 @@ public class TestBlockDeletingService { .setChunkName(chunk.getAbsolutePath()) .setLen(0) .setOffset(0) - .setChecksumData( - ContainerProtos.ChecksumData.getDefaultInstance()) + .setChecksumData(Checksum.getNoChecksumDataProto()) .build(); chunks.add(info); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java index 9df4249..2973a76 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.container.common.helpers; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.ozone.common.Checksum; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -41,7 +42,11 @@ public class TestBlockData { static ContainerProtos.ChunkInfo buildChunkInfo(String name, long offset, long len) { return ContainerProtos.ChunkInfo.newBuilder() - .setChunkName(name).setOffset(offset).setLen(len).build(); + .setChunkName(name) + .setOffset(offset) + .setLen(len) + .setChecksumData(Checksum.getNoChecksumDataProto()) + .build(); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java index 26a4ac1..a587ca3 100644 --- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java +++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java @@ -132,9 +132,10 @@ public final class DistributedStorageHandler implements StorageHandler { OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS); - int configuredChecksumSize = conf.getInt( + int configuredChecksumSize = (int) conf.getStorageSize( OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM, - OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT); + OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT, + StorageUnit.BYTES); int checksumSize; if(configuredChecksumSize < OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
