This is an automated email from the ASF dual-hosted git repository.
mthakur pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new f9f7b292eb8 HADOOP-19548: [ABFS] Fix Logging in FSDataInputStream to
Mention Correct Buffersize (#7642)
f9f7b292eb8 is described below
commit f9f7b292eb8604409efc7a7f8c751cceabdb0f40
Author: manika137 <80244229+manika...@users.noreply.github.com>
AuthorDate: Tue Apr 29 22:27:00 2025 +0530
HADOOP-19548: [ABFS] Fix Logging in FSDataInputStream to Mention Correct
Buffersize (#7642)
Contributed by: Manika Joshi
---
.../dev-support/testrun-scripts/configsupport.sh | 2 +-
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 23 ++++-------------
.../fs/azurebfs/ITestFileSystemProperties.java | 29 ++++++++++++++++++++++
3 files changed, 35 insertions(+), 19 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/configsupport.sh
b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/configsupport.sh
index d88f3df27fc..ab73b9d4db0 100755
--- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/configsupport.sh
+++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/configsupport.sh
@@ -115,7 +115,7 @@ promptNamespaceType() {
printf "* In the left-hand menu, select 'Overview' section and look for
'Properties'. \n"
printf "* Under 'Blob service', check if 'Hierarchical namespace' is
enabled or disabled. \n"
echo "$contactTeamMsg"
- select namespaceType in "HNS" "NonHNS"
+ select namespaceType in "HNS" "NonHNS (FNS)"
do
case $namespaceType in
HNS)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 0f707819271..d8074be5d17 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.fs.azurebfs;
-import javax.annotation.Nullable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -42,6 +41,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,22 +118,7 @@
import static
org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL;
import static
org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT;
import static
org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_APPEND;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_CREATE;
-import static
org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_CREATE_NON_RECURSIVE;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_DELETE;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_EXIST;
-import static
org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_GET_DELEGATION_TOKEN;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_GET_FILE_STATUS;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_LIST_STATUS;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_MKDIRS;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_OPEN;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_RENAME;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.DIRECTORIES_CREATED;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.DIRECTORIES_DELETED;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.ERROR_IGNORED;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.FILES_CREATED;
-import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.FILES_DELETED;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*;
import static
org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CPK_IN_NON_HNS_ACCOUNT_ERROR_MESSAGE;
import static org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType.DFS;
import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.DATA_BLOCKS_BUFFER;
@@ -365,8 +350,10 @@ public void registerListener(Listener listener1) {
@Override
public FSDataInputStream open(final Path path, final int bufferSize) throws
IOException {
- LOG.debug("AzureBlobFileSystem.open path: {} bufferSize: {}", path,
bufferSize);
// bufferSize is unused.
+ LOG.debug(
+ "AzureBlobFileSystem.open path: {} bufferSize as configured in
'fs.azure.read.request.size': {}",
+ path, abfsStore.getAbfsConfiguration().getReadBufferSize());
return open(path, Optional.empty());
}
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
index 0ccef2e6ccb..3f2a4fe9880 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
@@ -20,14 +20,18 @@
import java.util.Hashtable;
+import org.assertj.core.api.Assertions;
import org.junit.Test;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB;
+
/**
* Test FileSystemProperties.
*/
@@ -135,4 +139,29 @@ public void testSetFileSystemProperties() throws Exception
{
assertEquals(properties, fetchedProperties);
}
+
+ @Test
+ //Test to verify buffersize remains the same as set in the configuration,
irrespective of the parameter passed to FSDataInputStream
+ public void testBufferSizeSet() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ AbfsConfiguration abfsConfig = fs.getAbfsStore().getAbfsConfiguration();
+ int bufferSizeConfig = 6 * ONE_MB;
+ int bufferSizeArg = 10 * ONE_MB;
+
+ Path testPath = path(TEST_PATH);
+ fs.create(testPath);
+
+ abfsConfig.setReadBufferSize(bufferSizeConfig);
+ FSDataInputStream inputStream = fs.open(testPath, bufferSizeArg);
+ AbfsInputStream abfsInputStream
+ = (AbfsInputStream) inputStream.getWrappedStream();
+ int actualBufferSize = abfsInputStream.getBufferSize();
+
+ Assertions.assertThat(actualBufferSize)
+ .describedAs("Buffer size should be set to the value in the
configuration")
+ .isEqualTo(bufferSizeConfig);
+ Assertions.assertThat(actualBufferSize)
+ .describedAs("Buffer size should not be set to the value passed as
argument")
+ .isNotEqualTo(bufferSizeArg);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
