adoroszlai commented on code in PR #10961:
URL: https://github.com/apache/ozone/pull/10961#discussion_r3729812314
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestStreamRead.java:
##########
@@ -127,85 +145,90 @@ void testReadKey256k() throws Exception {
}
void runTestReadKey(SizeInBytes keySize, SizeInBytes bytesPerChecksum)
throws Exception {
- System.out.println("cluster starting ...");
- try (MiniOzoneCluster cluster = newCluster(bytesPerChecksum.getSizeInt()))
{
- cluster.waitForClusterToBeReady();
- System.out.println("cluster ready");
-
- final List<HddsDatanodeService> datanodes = cluster.getHddsDatanodes();
- assertEquals(1, datanodes.size());
- final HddsDatanodeService datanode = datanodes.get(0);
-
- OzoneConfiguration conf = cluster.getConf();
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamReadBlock(true);
- final OzoneConfiguration steamReadConf = new OzoneConfiguration(conf);
- steamReadConf.setFromObject(clientConfig);
-
- clientConfig.setStreamReadBlock(false);
- final OzoneConfiguration nonSteamReadConf = new OzoneConfiguration(conf);
- nonSteamReadConf.setFromObject(clientConfig);
-
- final SizeInBytes[] bufferSizes = {
- SizeInBytes.valueOf("32M"),
- SizeInBytes.valueOf("8M"),
- SizeInBytes.valueOf("1M"),
- SizeInBytes.valueOf("4k"),
- };
-
- try (OzoneClient streamReadClient =
OzoneClientFactory.getRpcClient(steamReadConf);
- OzoneClient nonStreamReadClient =
OzoneClientFactory.getRpcClient(nonSteamReadConf)) {
- final BucketForTesting testBucket =
BucketForTesting.newBuilder(streamReadClient).build();
- final String volume = testBucket.delegate().getVolumeName();
- final String bucket = testBucket.delegate().getName();
- final String keyName = "key0";
-
- // get the client ready by writing a dummy key
- createKey(testBucket.delegate(), DUMMY_KEY, SizeInBytes.ONE_KB,
SizeInBytes.ONE_KB);
-
- for (SizeInBytes bufferSize : bufferSizes) {
- // create key
-
System.out.println("---------------------------------------------------------");
- createKey(testBucket.delegate(), keyName, keySize, bufferSize);
-
- // get block file and generate md5
- final OmKeyInfo info =
nonStreamReadClient.getProxy().getKeyInfo(volume, bucket, keyName, false);
- final List<OmKeyLocationInfo> locations =
info.getLatestVersionLocations().getLocationList();
- assertEquals(1, locations.size());
- final BlockID blockId = locations.get(0).getBlockID();
- final ContainerData containerData =
datanode.getDatanodeStateMachine().getContainer().getContainerSet()
- .getContainer(blockId.getContainerID()).getContainerData();
- final File blockFile =
ContainerLayoutVersion.FILE_PER_BLOCK.getChunkFile(containerData, blockId,
null);
- assertTrue(blockFile.exists());
- assertEquals(BLOCK_SIZE, blockFile.length());
- final String expectedMd5 = generateMd5(keySize, SizeInBytes.ONE_MB,
blockFile);
-
- // run tests
-
System.out.println("---------------------------------------------------------");
- System.out.printf("%s with %s bytes and %s bytesPerChecksum%n",
- keyName, keySize, bytesPerChecksum);
-
- final CheckedBiConsumer<SizeInBytes, String, Exception> streamRead =
(readBufferSize, md5)
- -> streamRead(keySize, readBufferSize, md5, testBucket, keyName);
- final CheckedBiConsumer<SizeInBytes, String, Exception>
nonStreamRead = (readBufferSize, md5)
- -> nonStreamRead(keySize, readBufferSize, md5,
nonStreamReadClient, volume, bucket, keyName);
- final CheckedBiConsumer<SizeInBytes, String, Exception> fileRead =
(readBufferSize, md5)
- -> fileRead(keySize, readBufferSize, md5, blockFile);
- final List<CheckedBiConsumer<SizeInBytes, String, Exception>>
operations
- = Arrays.asList(streamRead, nonStreamRead, fileRead);
- Collections.shuffle(operations);
-
- for (CheckedBiConsumer<SizeInBytes, String, Exception> op :
operations) {
- for (int i = 0; i < 5; i++) {
- op.accept(bufferSize, null);
- }
- op.accept(bufferSize, expectedMd5);
+ final List<HddsDatanodeService> datanodes = cluster.getHddsDatanodes();
+ assertEquals(1, datanodes.size());
+ final HddsDatanodeService datanode = datanodes.get(0);
+
+ OzoneConfiguration conf = cluster.getConf();
+ OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
+ clientConfig.setBytesPerChecksum(bytesPerChecksum.getSizeInt());
+ clientConfig.setStreamReadBlock(true);
+ final OzoneConfiguration steamReadConf = new OzoneConfiguration(conf);
+ steamReadConf.setFromObject(clientConfig);
+
+ clientConfig.setStreamReadBlock(false);
+ final OzoneConfiguration nonSteamReadConf = new OzoneConfiguration(conf);
+ nonSteamReadConf.setFromObject(clientConfig);
+
+ final SizeInBytes[] bufferSizes = {
+ SizeInBytes.valueOf("32M"),
+ SizeInBytes.valueOf("8M"),
+ SizeInBytes.valueOf("1M"),
+ SizeInBytes.valueOf("4k"),
+ };
+
+ try (OzoneClient streamReadClient =
OzoneClientFactory.getRpcClient(steamReadConf);
+ OzoneClient nonStreamReadClient =
OzoneClientFactory.getRpcClient(nonSteamReadConf)) {
+
+ assertClientBytesPerChecksum(bytesPerChecksum, streamReadClient);
+ assertClientBytesPerChecksum(bytesPerChecksum, nonStreamReadClient);
+
+ final BucketForTesting testBucket =
BucketForTesting.newBuilder(streamReadClient).build();
+ final String volume = testBucket.delegate().getVolumeName();
+ final String bucket = testBucket.delegate().getName();
+ final String keyName = "key0";
+
+ // get the client ready by writing a dummy key
+ createKey(testBucket.delegate(), DUMMY_KEY, SizeInBytes.ONE_KB,
SizeInBytes.ONE_KB);
+
+ for (SizeInBytes bufferSize : bufferSizes) {
+ // create key
+
System.out.println("---------------------------------------------------------");
+ createKey(testBucket.delegate(), keyName, keySize, bufferSize);
+
+ // get block file and generate md5
+ final OmKeyInfo info =
nonStreamReadClient.getProxy().getKeyInfo(volume, bucket, keyName, false);
+ final List<OmKeyLocationInfo> locations =
info.getLatestVersionLocations().getLocationList();
+ assertEquals(1, locations.size());
+ final BlockID blockId = locations.get(0).getBlockID();
+ final ContainerData containerData =
datanode.getDatanodeStateMachine().getContainer().getContainerSet()
+ .getContainer(blockId.getContainerID()).getContainerData();
+ final File blockFile =
ContainerLayoutVersion.FILE_PER_BLOCK.getChunkFile(containerData, blockId,
null);
+ assertTrue(blockFile.exists());
+ assertEquals(BLOCK_SIZE, blockFile.length());
+ final String expectedMd5 = generateMd5(keySize, SizeInBytes.ONE_MB,
blockFile);
Review Comment:
I get the following error:
```
NoSuchAlgorithmException: CRC32 MessageDigest not available
```
I also tried SHA-256, but it's even slower.
BTW, MD5 is used many places, so if there is room for improvement, it should
be applied globally.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
