snvijaya commented on a change in pull request #2368:
URL: https://github.com/apache/hadoop/pull/2368#discussion_r504650837
##########
File path:
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
##########
@@ -447,4 +490,168 @@ public void testReadAheadManagerForSuccessfulReadAhead()
throws Exception {
checkEvictedStatus(inputStream, 0, true);
}
+ /**
+ * Test readahead with different config settings for request request size and
+ * readAhead block size
+ * @throws Exception
+ */
+ @Test
+ public void testDiffReadRequestSizeAndRAHBlockSize() throws Exception {
+ // Set requestRequestSize = 4MB and readAheadBufferSize=8MB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FOUR_MB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ testReadAheadConfigs(FOUR_MB, TEST_READAHEAD_DEPTH_4, false, EIGHT_MB);
+
+ // Test for requestRequestSize =16KB and readAheadBufferSize=16KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(SIXTEEN_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ AbfsInputStream inputStream = testReadAheadConfigs(SIXTEEN_KB,
+ TEST_READAHEAD_DEPTH_2, true, SIXTEEN_KB);
+ testReadAheads(inputStream, SIXTEEN_KB, SIXTEEN_KB);
+
+ // Test for requestRequestSize =16KB and readAheadBufferSize=48KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FORTY_EIGHT_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ inputStream = testReadAheadConfigs(SIXTEEN_KB, TEST_READAHEAD_DEPTH_2,
true,
+ FORTY_EIGHT_KB);
+ testReadAheads(inputStream, SIXTEEN_KB, FORTY_EIGHT_KB);
+
+ // Test for requestRequestSize =48KB and readAheadBufferSize=16KB
+ ReadBufferManager.getBufferManager()
+ .testResetReadBufferManager(FORTY_EIGHT_KB,
INCREASED_READ_BUFFER_AGE_THRESHOLD);
+ inputStream = testReadAheadConfigs(FORTY_EIGHT_KB, TEST_READAHEAD_DEPTH_2,
+ true,
+ SIXTEEN_KB);
+ testReadAheads(inputStream, FORTY_EIGHT_KB, SIXTEEN_KB);
+ }
+
+
+ private void testReadAheads(AbfsInputStream inputStream,
+ int readRequestSize,
+ int readAheadRequestSize)
+ throws Exception {
+ if (readRequestSize > readAheadRequestSize) {
+ readAheadRequestSize = readRequestSize;
+ }
+
+ byte[] firstReadBuffer = new byte[readRequestSize];
+ byte[] secondReadBuffer = new byte[readAheadRequestSize];
+
+ // get the expected bytes to compare
+ byte[] expectedFirstReadAheadBufferContents = new byte[readRequestSize];
+ byte[] expectedSecondReadAheadBufferContents = new
byte[readAheadRequestSize];
+ getExpectedBufferData(0, readRequestSize,
expectedFirstReadAheadBufferContents);
+ getExpectedBufferData(readRequestSize, readAheadRequestSize,
+ expectedSecondReadAheadBufferContents);
+
+ assertTrue("Read should be of exact requested size",
+ inputStream.read(firstReadBuffer, 0, readRequestSize) ==
readRequestSize);
+ assertTrue("Data mismatch found in RAH1",
+ Arrays.equals(firstReadBuffer,
+ expectedFirstReadAheadBufferContents));
+
+
+ assertTrue("Read should be of exact requested size",
+ inputStream.read(secondReadBuffer, 0, readAheadRequestSize) ==
readAheadRequestSize);
+ assertTrue("Data mismatch found in RAH2",
+ Arrays.equals(secondReadBuffer,
+ expectedSecondReadAheadBufferContents));
+ }
+
+ public AbfsInputStream testReadAheadConfigs(int readRequestSize,
+ int readAheadQueueDepth,
+ boolean alwaysReadBufferSizeEnabled,
+ int readAheadBlockSize) throws Exception {
+ Configuration
+ config = new Configuration(
+ this.getRawConfiguration());
+ config.set("fs.azure.read.request.size",
Integer.toString(readRequestSize));
+ config.set("fs.azure.readaheadqueue.depth",
+ Integer.toString(readAheadQueueDepth));
+ config.set("fs.azure.read.alwaysReadBufferSize",
+ Boolean.toString(alwaysReadBufferSizeEnabled));
+ config.set("fs.azure.read.readahead.blocksize",
+ Integer.toString(readAheadBlockSize));
+ if (readRequestSize > readAheadBlockSize) {
+ readAheadBlockSize = readRequestSize;
+ }
+
+ Path testPath = new Path(
+ "/testReadAheadConfigs");
+ final AzureBlobFileSystem fs = createTestFile(testPath,
+ ALWAYS_READ_BUFFER_SIZE_TEST_FILE_SIZE, config);
+ byte[] byteBuffer = new byte[ONE_MB];
+ AbfsInputStream inputStream = this.getAbfsStore(fs)
+ .openFileForRead(testPath, null);
+
+ assertEquals("Unexpected AbfsInputStream buffer size", readRequestSize,
+ inputStream.getBufferSize());
+ assertEquals("Unexpected ReadAhead queue depth", readAheadQueueDepth,
+ inputStream.getReadAheadQueueDepth());
+ assertEquals("Unexpected AlwaysReadBufferSize settings",
+ alwaysReadBufferSizeEnabled,
+ inputStream.shouldAlwaysReadBufferSize());
+ assertEquals("Unexpected readAhead block size", readAheadBlockSize,
+ ReadBufferManager.getBufferManager().getReadAheadBlockSize());
+
+ return inputStream;
+ }
+
+ private void getExpectedBufferData(int offset, int length, byte[] b) {
+ boolean startFillingIn = false;
+ int indexIntoBuffer = 0;
+ char character = 'a';
+
+ for (int i = 0; i < (offset + length); i++) {
+ if (i == offset) {
+ startFillingIn = true;
+ }
+
+ if ((startFillingIn) && (indexIntoBuffer < length)) {
+ b[indexIntoBuffer] = (byte) character;
+ indexIntoBuffer++;
+ }
+
+ character = (character == 'z') ? 'a' : (char) ((int) character + 1);
+ }
+ }
+
+ private AzureBlobFileSystem createTestFile(Path testFilePath, long
testFileSize,
Review comment:
I need the file created to also have a specific pattern of data ingested
and have another method which needs to return the expected data content from
offset to range. This is being used to cross check with readahead buffer data.
Will retain for these tests. But thanks, hadnt observed the other APIs, will
default to them where there isnt a specific needs on the file content.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
