mukund-thakur commented on a change in pull request #2368:
URL: https://github.com/apache/hadoop/pull/2368#discussion_r504018690
##########
File path:
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
##########
@@ -448,15 +477,119 @@ public void testRandomReadPerformance() throws Exception
{
ratio < maxAcceptableRatio);
}
+ /**
+ * With this test we should see a full buffer read being triggered in case
+ * alwaysReadBufferSize is on, else only the requested buffer size.
+ * Hence a seek done few bytes away from last read position will trigger
+ * a network read when alwaysReadBufferSize is off, whereas it will return
+ * from the internal buffer when it is on.
+ * Reading a full buffer size is the Gen1 behaviour.
+ * @throws Throwable
+ */
+ @Test
+ public void testAlwaysReadBufferSizeConfig() throws Throwable {
+ testAlwaysReadBufferSizeConfig(false);
+ testAlwaysReadBufferSizeConfig(true);
+ }
+
+ private void assertStatistics(AzureBlobFileSystem fs,
Review comment:
Why creating a new method here if we are just doing a passthrough?
##########
File path:
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
##########
@@ -447,4 +490,168 @@ public void testReadAheadManagerForSuccessfulReadAhead()
throws Exception {
checkEvictedStatus(inputStream, 0, true);
}
+ /**
+ * Test readahead with different config settings for request request size and
+ * readAhead block size
+ * @throws Exception
+ */
+ @Test
+ public void testDiffReadRequestSizeAndRAHBlockSize() throws Exception {
+ // Set requestRequestSize = 4MB and readAheadBufferSize=8MB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FOUR_MB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ testReadAheadConfigs(FOUR_MB, TEST_READAHEAD_DEPTH_4, false, EIGHT_MB);
+
+ // Test for requestRequestSize =16KB and readAheadBufferSize=16KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(SIXTEEN_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ AbfsInputStream inputStream = testReadAheadConfigs(SIXTEEN_KB,
+ TEST_READAHEAD_DEPTH_2, true, SIXTEEN_KB);
+ testReadAheads(inputStream, SIXTEEN_KB, SIXTEEN_KB);
+
+ // Test for requestRequestSize =16KB and readAheadBufferSize=48KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FORTY_EIGHT_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ inputStream = testReadAheadConfigs(SIXTEEN_KB, TEST_READAHEAD_DEPTH_2,
true,
+ FORTY_EIGHT_KB);
+ testReadAheads(inputStream, SIXTEEN_KB, FORTY_EIGHT_KB);
+
+ // Test for requestRequestSize =48KB and readAheadBufferSize=16KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FORTY_EIGHT_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ inputStream = testReadAheadConfigs(FORTY_EIGHT_KB, TEST_READAHEAD_DEPTH_2,
+ true,
+ SIXTEEN_KB);
+ testReadAheads(inputStream, FORTY_EIGHT_KB, SIXTEEN_KB);
+ }
+
+
+ private void testReadAheads(AbfsInputStream inputStream,
+ int readRequestSize,
+ int readAheadRequestSize)
+ throws Exception {
+ if (readRequestSize > readAheadRequestSize) {
+ readAheadRequestSize = readRequestSize;
+ }
+
+ byte[] firstReadBuffer = new byte[readRequestSize];
+ byte[] secondReadBuffer = new byte[readAheadRequestSize];
+
+ // get the expected bytes to compare
+ byte[] expectedFirstReadAheadBufferContents = new byte[readRequestSize];
+ byte[] expectedSecondReadAheadBufferContents = new
byte[readAheadRequestSize];
+ getExpectedBufferData(0, readRequestSize,
expectedFirstReadAheadBufferContents);
+ getExpectedBufferData(readRequestSize, readAheadRequestSize,
+ expectedSecondReadAheadBufferContents);
+
+ assertTrue("Read should be of exact requested size",
+ inputStream.read(firstReadBuffer, 0, readRequestSize) ==
readRequestSize);
+ assertTrue("Data mismatch found in RAH1",
+ Arrays.equals(firstReadBuffer,
+ expectedFirstReadAheadBufferContents));
+
+
+ assertTrue("Read should be of exact requested size",
+ inputStream.read(secondReadBuffer, 0, readAheadRequestSize) ==
readAheadRequestSize);
+ assertTrue("Data mismatch found in RAH2",
Review comment:
Better to use assert equals here inspite of assertTrue no?
##########
File path:
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
##########
@@ -447,4 +490,168 @@ public void testReadAheadManagerForSuccessfulReadAhead()
throws Exception {
checkEvictedStatus(inputStream, 0, true);
}
+ /**
+ * Test readahead with different config settings for request request size and
+ * readAhead block size
+ * @throws Exception
+ */
+ @Test
+ public void testDiffReadRequestSizeAndRAHBlockSize() throws Exception {
+ // Set requestRequestSize = 4MB and readAheadBufferSize=8MB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FOUR_MB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ testReadAheadConfigs(FOUR_MB, TEST_READAHEAD_DEPTH_4, false, EIGHT_MB);
+
+ // Test for requestRequestSize =16KB and readAheadBufferSize=16KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(SIXTEEN_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ AbfsInputStream inputStream = testReadAheadConfigs(SIXTEEN_KB,
+ TEST_READAHEAD_DEPTH_2, true, SIXTEEN_KB);
+ testReadAheads(inputStream, SIXTEEN_KB, SIXTEEN_KB);
+
+ // Test for requestRequestSize =16KB and readAheadBufferSize=48KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FORTY_EIGHT_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ inputStream = testReadAheadConfigs(SIXTEEN_KB, TEST_READAHEAD_DEPTH_2,
true,
+ FORTY_EIGHT_KB);
+ testReadAheads(inputStream, SIXTEEN_KB, FORTY_EIGHT_KB);
+
+ // Test for requestRequestSize =48KB and readAheadBufferSize=16KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FORTY_EIGHT_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ inputStream = testReadAheadConfigs(FORTY_EIGHT_KB, TEST_READAHEAD_DEPTH_2,
+ true,
+ SIXTEEN_KB);
+ testReadAheads(inputStream, FORTY_EIGHT_KB, SIXTEEN_KB);
+ }
+
+
+ private void testReadAheads(AbfsInputStream inputStream,
+ int readRequestSize,
+ int readAheadRequestSize)
+ throws Exception {
+ if (readRequestSize > readAheadRequestSize) {
+ readAheadRequestSize = readRequestSize;
+ }
+
+ byte[] firstReadBuffer = new byte[readRequestSize];
+ byte[] secondReadBuffer = new byte[readAheadRequestSize];
+
+ // get the expected bytes to compare
+ byte[] expectedFirstReadAheadBufferContents = new byte[readRequestSize];
+ byte[] expectedSecondReadAheadBufferContents = new
byte[readAheadRequestSize];
+ getExpectedBufferData(0, readRequestSize,
expectedFirstReadAheadBufferContents);
+ getExpectedBufferData(readRequestSize, readAheadRequestSize,
+ expectedSecondReadAheadBufferContents);
+
+ assertTrue("Read should be of exact requested size",
+ inputStream.read(firstReadBuffer, 0, readRequestSize) ==
readRequestSize);
+ assertTrue("Data mismatch found in RAH1",
+ Arrays.equals(firstReadBuffer,
+ expectedFirstReadAheadBufferContents));
+
+
+ assertTrue("Read should be of exact requested size",
+ inputStream.read(secondReadBuffer, 0, readAheadRequestSize) ==
readAheadRequestSize);
+ assertTrue("Data mismatch found in RAH2",
+ Arrays.equals(secondReadBuffer,
+ expectedSecondReadAheadBufferContents));
+ }
+
+ public AbfsInputStream testReadAheadConfigs(int readRequestSize,
+ int readAheadQueueDepth,
+ boolean alwaysReadBufferSizeEnabled,
+ int readAheadBlockSize) throws Exception {
+ Configuration
+ config = new Configuration(
+ this.getRawConfiguration());
+ config.set("fs.azure.read.request.size",
Integer.toString(readRequestSize));
+ config.set("fs.azure.readaheadqueue.depth",
+ Integer.toString(readAheadQueueDepth));
+ config.set("fs.azure.read.alwaysReadBufferSize",
+ Boolean.toString(alwaysReadBufferSizeEnabled));
+ config.set("fs.azure.read.readahead.blocksize",
+ Integer.toString(readAheadBlockSize));
+ if (readRequestSize > readAheadBlockSize) {
+ readAheadBlockSize = readRequestSize;
+ }
+
+ Path testPath = new Path(
+ "/testReadAheadConfigs");
+ final AzureBlobFileSystem fs = createTestFile(testPath,
+ ALWAYS_READ_BUFFER_SIZE_TEST_FILE_SIZE, config);
+ byte[] byteBuffer = new byte[ONE_MB];
+ AbfsInputStream inputStream = this.getAbfsStore(fs)
+ .openFileForRead(testPath, null);
+
+ assertEquals("Unexpected AbfsInputStream buffer size", readRequestSize,
+ inputStream.getBufferSize());
+ assertEquals("Unexpected ReadAhead queue depth", readAheadQueueDepth,
+ inputStream.getReadAheadQueueDepth());
+ assertEquals("Unexpected AlwaysReadBufferSize settings",
+ alwaysReadBufferSizeEnabled,
+ inputStream.shouldAlwaysReadBufferSize());
+ assertEquals("Unexpected readAhead block size", readAheadBlockSize,
+ ReadBufferManager.getBufferManager().getReadAheadBlockSize());
+
+ return inputStream;
+ }
+
+ private void getExpectedBufferData(int offset, int length, byte[] b) {
+ boolean startFillingIn = false;
+ int indexIntoBuffer = 0;
+ char character = 'a';
+
+ for (int i = 0; i < (offset + length); i++) {
+ if (i == offset) {
+ startFillingIn = true;
+ }
+
+ if ((startFillingIn) && (indexIntoBuffer < length)) {
+ b[indexIntoBuffer] = (byte) character;
+ indexIntoBuffer++;
+ }
+
+ character = (character == 'z') ? 'a' : (char) ((int) character + 1);
+ }
+ }
+
+ private AzureBlobFileSystem createTestFile(Path testFilePath, long
testFileSize,
Review comment:
See if you can reuse the data generation and new file creation code from
ContractTestUtils.dataset() and ContractTestUtils.createFile.
##########
File path:
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
##########
@@ -447,4 +490,168 @@ public void testReadAheadManagerForSuccessfulReadAhead()
throws Exception {
checkEvictedStatus(inputStream, 0, true);
}
+ /**
+ * Test readahead with different config settings for request request size and
+ * readAhead block size
+ * @throws Exception
+ */
+ @Test
+ public void testDiffReadRequestSizeAndRAHBlockSize() throws Exception {
+ // Set requestRequestSize = 4MB and readAheadBufferSize=8MB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FOUR_MB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ testReadAheadConfigs(FOUR_MB, TEST_READAHEAD_DEPTH_4, false, EIGHT_MB);
+
+ // Test for requestRequestSize =16KB and readAheadBufferSize=16KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(SIXTEEN_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ AbfsInputStream inputStream = testReadAheadConfigs(SIXTEEN_KB,
+ TEST_READAHEAD_DEPTH_2, true, SIXTEEN_KB);
+ testReadAheads(inputStream, SIXTEEN_KB, SIXTEEN_KB);
+
+ // Test for requestRequestSize =16KB and readAheadBufferSize=48KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FORTY_EIGHT_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ inputStream = testReadAheadConfigs(SIXTEEN_KB, TEST_READAHEAD_DEPTH_2,
true,
+ FORTY_EIGHT_KB);
+ testReadAheads(inputStream, SIXTEEN_KB, FORTY_EIGHT_KB);
+
+ // Test for requestRequestSize =48KB and readAheadBufferSize=16KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FORTY_EIGHT_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ inputStream = testReadAheadConfigs(FORTY_EIGHT_KB, TEST_READAHEAD_DEPTH_2,
+ true,
+ SIXTEEN_KB);
+ testReadAheads(inputStream, FORTY_EIGHT_KB, SIXTEEN_KB);
+ }
+
+
+ private void testReadAheads(AbfsInputStream inputStream,
+ int readRequestSize,
+ int readAheadRequestSize)
+ throws Exception {
+ if (readRequestSize > readAheadRequestSize) {
+ readAheadRequestSize = readRequestSize;
+ }
+
+ byte[] firstReadBuffer = new byte[readRequestSize];
+ byte[] secondReadBuffer = new byte[readAheadRequestSize];
+
+ // get the expected bytes to compare
+ byte[] expectedFirstReadAheadBufferContents = new byte[readRequestSize];
+ byte[] expectedSecondReadAheadBufferContents = new
byte[readAheadRequestSize];
+ getExpectedBufferData(0, readRequestSize,
expectedFirstReadAheadBufferContents);
+ getExpectedBufferData(readRequestSize, readAheadRequestSize,
+ expectedSecondReadAheadBufferContents);
+
+ assertTrue("Read should be of exact requested size",
+ inputStream.read(firstReadBuffer, 0, readRequestSize) ==
readRequestSize);
+ assertTrue("Data mismatch found in RAH1",
+ Arrays.equals(firstReadBuffer,
Review comment:
AssertJ has rich api's to tackle these kind of assertions. Try that
example : Assertions.assertThat(list)
.hasSameElementsAs(list2)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
