yandrey321 commented on code in PR #10961:
URL: https://github.com/apache/ozone/pull/10961#discussion_r3729249550
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestStreamRead.java:
##########
@@ -127,85 +145,90 @@ void testReadKey256k() throws Exception {
}
void runTestReadKey(SizeInBytes keySize, SizeInBytes bytesPerChecksum)
throws Exception {
- System.out.println("cluster starting ...");
- try (MiniOzoneCluster cluster = newCluster(bytesPerChecksum.getSizeInt()))
{
- cluster.waitForClusterToBeReady();
- System.out.println("cluster ready");
-
- final List<HddsDatanodeService> datanodes = cluster.getHddsDatanodes();
- assertEquals(1, datanodes.size());
- final HddsDatanodeService datanode = datanodes.get(0);
-
- OzoneConfiguration conf = cluster.getConf();
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamReadBlock(true);
- final OzoneConfiguration steamReadConf = new OzoneConfiguration(conf);
- steamReadConf.setFromObject(clientConfig);
-
- clientConfig.setStreamReadBlock(false);
- final OzoneConfiguration nonSteamReadConf = new OzoneConfiguration(conf);
- nonSteamReadConf.setFromObject(clientConfig);
-
- final SizeInBytes[] bufferSizes = {
- SizeInBytes.valueOf("32M"),
- SizeInBytes.valueOf("8M"),
- SizeInBytes.valueOf("1M"),
- SizeInBytes.valueOf("4k"),
- };
-
- try (OzoneClient streamReadClient =
OzoneClientFactory.getRpcClient(steamReadConf);
- OzoneClient nonStreamReadClient =
OzoneClientFactory.getRpcClient(nonSteamReadConf)) {
- final BucketForTesting testBucket =
BucketForTesting.newBuilder(streamReadClient).build();
- final String volume = testBucket.delegate().getVolumeName();
- final String bucket = testBucket.delegate().getName();
- final String keyName = "key0";
-
- // get the client ready by writing a dummy key
- createKey(testBucket.delegate(), DUMMY_KEY, SizeInBytes.ONE_KB,
SizeInBytes.ONE_KB);
-
- for (SizeInBytes bufferSize : bufferSizes) {
- // create key
-
System.out.println("---------------------------------------------------------");
- createKey(testBucket.delegate(), keyName, keySize, bufferSize);
-
- // get block file and generate md5
- final OmKeyInfo info =
nonStreamReadClient.getProxy().getKeyInfo(volume, bucket, keyName, false);
- final List<OmKeyLocationInfo> locations =
info.getLatestVersionLocations().getLocationList();
- assertEquals(1, locations.size());
- final BlockID blockId = locations.get(0).getBlockID();
- final ContainerData containerData =
datanode.getDatanodeStateMachine().getContainer().getContainerSet()
- .getContainer(blockId.getContainerID()).getContainerData();
- final File blockFile =
ContainerLayoutVersion.FILE_PER_BLOCK.getChunkFile(containerData, blockId,
null);
- assertTrue(blockFile.exists());
- assertEquals(BLOCK_SIZE, blockFile.length());
- final String expectedMd5 = generateMd5(keySize, SizeInBytes.ONE_MB,
blockFile);
-
- // run tests
-
System.out.println("---------------------------------------------------------");
- System.out.printf("%s with %s bytes and %s bytesPerChecksum%n",
- keyName, keySize, bytesPerChecksum);
-
- final CheckedBiConsumer<SizeInBytes, String, Exception> streamRead =
(readBufferSize, md5)
- -> streamRead(keySize, readBufferSize, md5, testBucket, keyName);
- final CheckedBiConsumer<SizeInBytes, String, Exception>
nonStreamRead = (readBufferSize, md5)
- -> nonStreamRead(keySize, readBufferSize, md5,
nonStreamReadClient, volume, bucket, keyName);
- final CheckedBiConsumer<SizeInBytes, String, Exception> fileRead =
(readBufferSize, md5)
- -> fileRead(keySize, readBufferSize, md5, blockFile);
- final List<CheckedBiConsumer<SizeInBytes, String, Exception>>
operations
- = Arrays.asList(streamRead, nonStreamRead, fileRead);
- Collections.shuffle(operations);
-
- for (CheckedBiConsumer<SizeInBytes, String, Exception> op :
operations) {
- for (int i = 0; i < 5; i++) {
- op.accept(bufferSize, null);
- }
- op.accept(bufferSize, expectedMd5);
+ final List<HddsDatanodeService> datanodes = cluster.getHddsDatanodes();
+ assertEquals(1, datanodes.size());
+ final HddsDatanodeService datanode = datanodes.get(0);
+
+ OzoneConfiguration conf = cluster.getConf();
+ OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
+ clientConfig.setBytesPerChecksum(bytesPerChecksum.getSizeInt());
+ clientConfig.setStreamReadBlock(true);
+ final OzoneConfiguration steamReadConf = new OzoneConfiguration(conf);
+ steamReadConf.setFromObject(clientConfig);
+
+ clientConfig.setStreamReadBlock(false);
+ final OzoneConfiguration nonSteamReadConf = new OzoneConfiguration(conf);
+ nonSteamReadConf.setFromObject(clientConfig);
+
+ final SizeInBytes[] bufferSizes = {
+ SizeInBytes.valueOf("32M"),
+ SizeInBytes.valueOf("8M"),
+ SizeInBytes.valueOf("1M"),
+ SizeInBytes.valueOf("4k"),
+ };
+
+ try (OzoneClient streamReadClient =
OzoneClientFactory.getRpcClient(steamReadConf);
+ OzoneClient nonStreamReadClient =
OzoneClientFactory.getRpcClient(nonSteamReadConf)) {
+
+ assertClientBytesPerChecksum(bytesPerChecksum, streamReadClient);
+ assertClientBytesPerChecksum(bytesPerChecksum, nonStreamReadClient);
+
+ final BucketForTesting testBucket =
BucketForTesting.newBuilder(streamReadClient).build();
+ final String volume = testBucket.delegate().getVolumeName();
+ final String bucket = testBucket.delegate().getName();
+ final String keyName = "key0";
+
+ // get the client ready by writing a dummy key
+ createKey(testBucket.delegate(), DUMMY_KEY, SizeInBytes.ONE_KB,
SizeInBytes.ONE_KB);
+
+ for (SizeInBytes bufferSize : bufferSizes) {
+ // create key
+
System.out.println("---------------------------------------------------------");
+ createKey(testBucket.delegate(), keyName, keySize, bufferSize);
+
+ // get block file and generate md5
+ final OmKeyInfo info =
nonStreamReadClient.getProxy().getKeyInfo(volume, bucket, keyName, false);
+ final List<OmKeyLocationInfo> locations =
info.getLatestVersionLocations().getLocationList();
+ assertEquals(1, locations.size());
+ final BlockID blockId = locations.get(0).getBlockID();
+ final ContainerData containerData =
datanode.getDatanodeStateMachine().getContainer().getContainerSet()
+ .getContainer(blockId.getContainerID()).getContainerData();
+ final File blockFile =
ContainerLayoutVersion.FILE_PER_BLOCK.getChunkFile(containerData, blockId,
null);
+ assertTrue(blockFile.exists());
+ assertEquals(BLOCK_SIZE, blockFile.length());
+ final String expectedMd5 = generateMd5(keySize, SizeInBytes.ONE_MB,
blockFile);
Review Comment:
md5 is slow, we can use CRC32 for a checksum it should improve test
performance even further
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
