This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d1f5976 HADOOP-16699. Add verbose TRACE logging to ABFS.
d1f5976 is described below
commit d1f5976c004292f35b0fc5efbc86273becc4a304
Author: Sneha Vijayarajan <[email protected]>
AuthorDate: Tue Jan 7 18:05:47 2020 +0000
HADOOP-16699. Add verbose TRACE logging to ABFS.
Contributed by Sneha Vijayarajan,
Change-Id: Ic616a10406e6e9f11616c9cc05d8630ebbedaf65
---
.../org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 12 ++++++++++++
.../org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 4 ++++
.../apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java | 13 ++++++++++++-
.../org/apache/hadoop/fs/azurebfs/services/AbfsClient.java | 4 ++++
.../hadoop/fs/azurebfs/services/AbfsRestOperation.java | 4 +++-
5 files changed, 35 insertions(+), 2 deletions(-)
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index b9b09f0..81e4191 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -63,6 +63,9 @@ import
org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*;
import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.*;
@@ -76,6 +79,7 @@ public class AbfsConfiguration{
private final Configuration rawConfig;
private final String accountName;
private final boolean isSecure;
+ private static final Logger LOG =
LoggerFactory.getLogger(AbfsConfiguration.class);
@IntegerConfigurationValidatorAnnotation(ConfigurationKey =
AZURE_WRITE_BUFFER_SIZE,
MinValue = MIN_BUFFER_SIZE,
@@ -505,11 +509,13 @@ public class AbfsConfiguration{
String clientId =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
String clientSecret =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET);
tokenProvider = new ClientCredsTokenProvider(authEndpoint, clientId,
clientSecret);
+ LOG.trace("ClientCredsTokenProvider initialized");
} else if (tokenProviderClass == UserPasswordTokenProvider.class) {
String authEndpoint =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT);
String username =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_NAME);
String password =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD);
tokenProvider = new UserPasswordTokenProvider(authEndpoint,
username, password);
+ LOG.trace("UserPasswordTokenProvider initialized");
} else if (tokenProviderClass == MsiTokenProvider.class) {
String authEndpoint = getTrimmedPasswordString(
FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT,
@@ -522,6 +528,7 @@ public class AbfsConfiguration{
authority = appendSlashIfNeeded(authority);
tokenProvider = new MsiTokenProvider(authEndpoint, tenantGuid,
clientId, authority);
+ LOG.trace("MsiTokenProvider initialized");
} else if (tokenProviderClass == RefreshTokenBasedTokenProvider.class)
{
String authEndpoint = getTrimmedPasswordString(
FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT,
@@ -530,6 +537,7 @@ public class AbfsConfiguration{
String clientId =
getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID);
tokenProvider = new RefreshTokenBasedTokenProvider(authEndpoint,
clientId, refreshToken);
+ LOG.trace("RefreshTokenBasedTokenProvider initialized");
} else {
throw new IllegalArgumentException("Failed to initialize " +
tokenProviderClass);
}
@@ -554,7 +562,9 @@ public class AbfsConfiguration{
if (azureTokenProvider == null) {
throw new IllegalArgumentException("Failed to initialize " +
customTokenProviderClass);
}
+ LOG.trace("Initializing {}", customTokenProviderClass.getName());
azureTokenProvider.initialize(rawConfig, accountName);
+ LOG.trace("{} init complete", customTokenProviderClass.getName());
return new CustomTokenProviderAdapter(azureTokenProvider);
} catch(IllegalArgumentException e) {
throw e;
@@ -581,7 +591,9 @@ public class AbfsConfiguration{
@SuppressWarnings("unchecked")
Class<AbfsAuthorizer> authClass = (Class<AbfsAuthorizer>)
rawConfig.getClassByName(authClassName);
authorizer = authClass.getConstructor(new Class[]
{Configuration.class}).newInstance(rawConfig);
+ LOG.trace("Initializing {}", authClassName);
authorizer.init();
+ LOG.trace("{} init complete", authClassName);
}
} catch (
IllegalAccessException
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 2e88de2..fb591bb 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -107,6 +107,8 @@ public class AzureBlobFileSystem extends FileSystem {
this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
this.abfsStore = new AzureBlobFileSystemStore(uri, this.isSecureScheme(),
configuration);
+ LOG.trace("AzureBlobFileSystemStore init complete");
+
final AbfsConfiguration abfsConfiguration =
abfsStore.getAbfsConfiguration();
this.setWorkingDirectory(this.getHomeDirectory());
@@ -121,6 +123,7 @@ public class AzureBlobFileSystem extends FileSystem {
}
}
+ LOG.trace("Initiate check for delegation token manager");
if (UserGroupInformation.isSecurityEnabled()) {
this.delegationTokenEnabled =
abfsConfiguration.isDelegationTokenManagerEnabled();
@@ -137,6 +140,7 @@ public class AzureBlobFileSystem extends FileSystem {
// Initialize ABFS authorizer
//
this.authorizer = abfsConfiguration.getAbfsAuthorizer();
+ LOG.debug("Initializing AzureBlobFileSystem for {} complete", uri);
}
@Override
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 36ab57c..aff8111 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -148,8 +148,12 @@ public class AzureBlobFileSystemStore implements Closeable
{
} catch (IllegalAccessException exception) {
throw new FileSystemOperationUnhandledException(exception);
}
+
+ LOG.trace("AbfsConfiguration init complete");
+
this.userGroupInformation = UserGroupInformation.getCurrentUser();
this.userName = userGroupInformation.getShortUserName();
+ LOG.trace("UGI init complete");
if (!abfsConfiguration.getSkipUserGroupMetadataDuringInitialization()) {
try {
this.primaryUserGroup = userGroupInformation.getPrimaryGroupName();
@@ -161,6 +165,7 @@ public class AzureBlobFileSystemStore implements Closeable {
//Provide a default group name
this.primaryUserGroup = userName;
}
+ LOG.trace("primaryUserGroup is {}", this.primaryUserGroup);
this.azureAtomicRenameDirSet = new HashSet<>(Arrays.asList(
abfsConfiguration.getAzureAtomicRenameDirs().split(AbfsHttpConstants.COMMA)));
@@ -170,6 +175,7 @@ public class AzureBlobFileSystemStore implements Closeable {
this.abfsPerfTracker = new AbfsPerfTracker(fileSystemName, accountName,
this.abfsConfiguration);
initializeClient(uri, fileSystemName, accountName, useHttps);
this.identityTransformer = new
IdentityTransformer(abfsConfiguration.getRawConfiguration());
+ LOG.trace("IdentityTransformer init complete");
}
/**
@@ -296,6 +302,7 @@ public class AzureBlobFileSystemStore implements Closeable {
public void setFilesystemProperties(final Hashtable<String, String>
properties)
throws AzureBlobFileSystemException {
if (properties == null || properties.isEmpty()) {
+ LOG.trace("setFilesystemProperties no properties present");
return;
}
@@ -1116,6 +1123,7 @@ public class AzureBlobFileSystemStore implements
Closeable {
AccessTokenProvider tokenProvider = null;
if (abfsConfiguration.getAuthType(accountName) == AuthType.SharedKey) {
+ LOG.trace("Fetching SharedKey credentials");
int dotIndex = accountName.indexOf(AbfsHttpConstants.DOT);
if (dotIndex <= 0) {
throw new InvalidUriException(
@@ -1124,14 +1132,17 @@ public class AzureBlobFileSystemStore implements
Closeable {
creds = new SharedKeyCredentials(accountName.substring(0, dotIndex),
abfsConfiguration.getStorageAccountKey());
} else {
+ LOG.trace("Fetching token provider");
tokenProvider = abfsConfiguration.getTokenProvider();
ExtensionHelper.bind(tokenProvider, uri,
abfsConfiguration.getRawConfiguration());
}
- this.client = new AbfsClient(baseUrl, creds, abfsConfiguration,
+ LOG.trace("Initializing AbfsClient for {}", baseUrl);
+ this.client = new AbfsClient(baseUrl, creds, abfsConfiguration,
new ExponentialRetryPolicy(abfsConfiguration.getMaxIoRetries()),
tokenProvider, abfsPerfTracker);
+ LOG.trace("AbfsClient init complete");
}
private String getOctalNotation(FsPermission fsPermission) {
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
index f7eb4ed..374bde7 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
@@ -81,10 +81,14 @@ public class AbfsClient implements Closeable {
if (this.baseUrl.toString().startsWith(HTTPS_SCHEME)) {
try {
+ LOG.trace("Initializing DelegatingSSLSocketFactory with {} SSL "
+ + "Channel Mode",
this.abfsConfiguration.getPreferredSSLFactoryOption());
DelegatingSSLSocketFactory.initializeDefaultFactory(this.abfsConfiguration.getPreferredSSLFactoryOption());
sslProviderName =
DelegatingSSLSocketFactory.getDefaultFactory().getProviderName();
} catch (IOException e) {
// Suppress exception. Failure to init DelegatingSSLSocketFactory
would have only performance impact.
+ LOG.trace("NonCritFailure: DelegatingSSLSocketFactory Init failed : "
+ + "{}", e.getMessage());
}
}
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index f44094b..d2bda08 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
@@ -145,6 +145,8 @@ public class AbfsRestOperation {
throw new AbfsRestOperationException(result.getStatusCode(),
result.getStorageErrorCode(),
result.getStorageErrorMessage(), null, result);
}
+
+ LOG.trace("{} REST operation complete", operationType);
}
/**
@@ -210,7 +212,7 @@ public class AbfsRestOperation {
AbfsClientThrottlingIntercept.updateMetrics(operationType,
httpOperation);
}
- LOG.debug("HttpRequest: " + httpOperation.toString());
+ LOG.debug("HttpRequest: {}", httpOperation.toString());
if (client.getRetryPolicy().shouldRetry(retryCount,
httpOperation.getStatusCode())) {
return false;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
