This is an automated email from the ASF dual-hosted git repository.
anujmodi pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 8532d9e6f33 HADOOP-18325: [ABFS] Fix metric related test failures due
to missing config (#6847) (#7306)
8532d9e6f33 is described below
commit 8532d9e6f33cb6f2c7b88941d7350f3157e8448c
Author: Anmol Asrani <anmol.asrani...@gmail.com>
AuthorDate: Fri Jan 24 06:31:51 2025 +0000
HADOOP-18325: [ABFS] Fix metric related test failures due to missing config
(#6847) (#7306)
Fixing test failures when the needed configs for metric collection are not
set
Contributed by: Anmol Asrani
---
.../hadoop/fs/azurebfs/services/AbfsClient.java | 30 ++++++++++++----------
.../fs/azurebfs/ITestAbfsReadFooterMetrics.java | 19 ++++++++++++++
.../azurebfs/services/TestAbfsRestOperation.java | 10 ++++++++
3 files changed, 46 insertions(+), 13 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
index 2119b1b30c3..e5329a9a39a 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
@@ -42,6 +42,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.apache.hadoop.fs.azurebfs.constants.HttpOperationType;
@@ -247,21 +248,24 @@ private AbfsClient(final URL baseUrl,
this.isMetricCollectionStopped = new AtomicBoolean(false);
this.metricAnalysisPeriod = abfsConfiguration.getMetricAnalysisTimeout();
this.metricIdlePeriod = abfsConfiguration.getMetricIdleTimeout();
- if (!metricFormat.toString().equals("")) {
- isMetricCollectionEnabled = true;
- abfsCounters.initializeMetrics(metricFormat);
+ if (StringUtils.isNotEmpty(metricFormat.toString())) {
String metricAccountName = abfsConfiguration.getMetricAccount();
- int dotIndex = metricAccountName.indexOf(AbfsHttpConstants.DOT);
- if (dotIndex <= 0) {
- throw new InvalidUriException(
- metricAccountName + " - account name is not fully qualified.");
- }
String metricAccountKey = abfsConfiguration.getMetricAccountKey();
- try {
- metricSharedkeyCredentials = new
SharedKeyCredentials(metricAccountName.substring(0, dotIndex),
- metricAccountKey);
- } catch (IllegalArgumentException e) {
- throw new IOException("Exception while initializing metric credentials
" + e);
+ if (StringUtils.isNotEmpty(metricAccountName) &&
StringUtils.isNotEmpty(metricAccountKey)) {
+ isMetricCollectionEnabled = true;
+ abfsCounters.initializeMetrics(metricFormat);
+ int dotIndex = metricAccountName.indexOf(AbfsHttpConstants.DOT);
+ if (dotIndex <= 0) {
+ throw new InvalidUriException(
+ metricAccountName + " - account name is not fully qualified.");
+ }
+ try {
+ metricSharedkeyCredentials = new SharedKeyCredentials(
+ metricAccountName.substring(0, dotIndex),
+ metricAccountKey);
+ } catch (IllegalArgumentException e) {
+ throw new IOException("Exception while initializing metric
credentials ", e);
+ }
}
}
if (isMetricCollectionEnabled) {
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java
index 0071b90771c..90d769b56f4 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java
@@ -21,7 +21,10 @@
import static
org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO;
import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_BUFFER_SIZE;
import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_WRITE_BUFFER_SIZE;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME;
import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_FORMAT;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI;
import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_KB;
import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB;
@@ -30,6 +33,8 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.azurebfs.utils.MetricFormat;
+
+import org.junit.Assume;
import org.junit.Test;
import java.io.IOException;
@@ -47,6 +52,20 @@
public class ITestAbfsReadFooterMetrics extends AbstractAbfsScaleTest {
public ITestAbfsReadFooterMetrics() throws Exception {
+ checkPrerequisites();
+ }
+
+ private void checkPrerequisites(){
+ checkIfConfigIsSet(FS_AZURE_METRIC_ACCOUNT_NAME);
+ checkIfConfigIsSet(FS_AZURE_METRIC_ACCOUNT_KEY);
+ checkIfConfigIsSet(FS_AZURE_METRIC_URI);
+ }
+
+ private void checkIfConfigIsSet(String configKey){
+ AbfsConfiguration conf = getConfiguration();
+ String value = conf.get(configKey);
+ Assume.assumeTrue(configKey + " config is mandatory for the test to run",
+ value != null && value.trim().length() > 1);
}
private static final String TEST_PATH = "/testfile";
diff --git
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java
index 1c6ee6e7dc9..e5fcf9e71ed 100644
---
a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java
+++
b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java
@@ -24,7 +24,10 @@
import org.apache.hadoop.fs.azurebfs.utils.MetricFormat;
import org.junit.Test;
import static
org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_DELETE;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME;
import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_FORMAT;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI;
import static
org.apache.hadoop.fs.azurebfs.services.AbfsRestOperationType.DeletePath;
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
@@ -39,6 +42,12 @@ public class TestAbfsRestOperation extends
public TestAbfsRestOperation() throws Exception {
}
+ private void checkPrerequisites() {
+ assumeValidTestConfigPresent(getRawConfiguration(),
FS_AZURE_METRIC_ACCOUNT_NAME);
+ assumeValidTestConfigPresent(getRawConfiguration(),
FS_AZURE_METRIC_ACCOUNT_KEY);
+ assumeValidTestConfigPresent(getRawConfiguration(), FS_AZURE_METRIC_URI);
+ }
+
/**
* Test for backoff retry metrics.
*
@@ -49,6 +58,7 @@ public TestAbfsRestOperation() throws Exception {
*/
@Test
public void testBackoffRetryMetrics() throws Exception {
+ checkPrerequisites();
// Create an AzureBlobFileSystem instance.
final Configuration configuration = getRawConfiguration();
configuration.set(FS_AZURE_METRIC_FORMAT,
String.valueOf(MetricFormat.INTERNAL_BACKOFF_METRIC_FORMAT));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
