HADOOP-14518. Customize User-Agent header sent in HTTP/HTTPS requests by WASB. Contributed by Georgi Chalakov.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2921e51 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2921e51 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2921e51 Branch: refs/heads/YARN-5734 Commit: f2921e51f0fe613abce0a9f415a0d8ab6144aa6e Parents: c98201b Author: Jitendra Pandey <jiten...@apache.org> Authored: Mon Jul 24 13:59:27 2017 -0700 Committer: Jitendra Pandey <jiten...@apache.org> Committed: Mon Jul 24 23:01:01 2017 -0700 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 10 +- .../conf/TestCommonConfigurationFields.java | 1 + .../fs/azure/AzureNativeFileSystemStore.java | 144 +++++++++++-------- .../hadoop-azure/src/site/markdown/index.md | 13 ++ .../fs/azure/TestWasbUriAndConfiguration.java | 48 +++++++ .../src/test/resources/azure-test.xml | 5 + 6 files changed, 162 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2921e51/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 68b0a9d..d5ddc7f 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -499,7 +499,15 @@ name to use for the service when the client wishes to make an RPC call. </description> </property> - + <property> + <name>fs.azure.user.agent.prefix</name> + <value>unknown</value> + <description> + WASB passes User-Agent header to the Azure back-end. The default value + contains WASB version, Java Runtime version, Azure Client library version, + and the value of the configuration option fs.azure.user.agent.prefix. + </description> + </property> <property> <name>hadoop.security.uid.cache.secs</name> http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2921e51/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index 593254eb..ef74cba 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -116,6 +116,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { xmlPropsToSkipCompare.add("fs.azure.secure.mode"); xmlPropsToSkipCompare.add("fs.azure.authorization"); xmlPropsToSkipCompare.add("fs.azure.authorization.caching.enable"); + xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix"); // Deprecated properties. These should eventually be removed from the // class. http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2921e51/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 6b6f07a..7c198af 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -57,6 +57,7 @@ import org.apache.hadoop.fs.azure.metrics.ResponseReceivedMetricUpdater; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.VersionInfo; import org.eclipse.jetty.util.ajax.JSON; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,6 +72,10 @@ import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; import com.microsoft.azure.storage.StorageErrorCode; import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.Constants; +import com.microsoft.azure.storage.StorageEvent; +import com.microsoft.azure.storage.core.BaseRequest; +import com.microsoft.azure.storage.SendingRequestEvent; import com.microsoft.azure.storage.blob.BlobListingDetails; import com.microsoft.azure.storage.blob.BlobProperties; import com.microsoft.azure.storage.blob.BlobRequestOptions; @@ -83,13 +88,13 @@ import com.microsoft.azure.storage.core.Utility; /** * Core implementation of Windows Azure Filesystem for Hadoop. - * Provides the bridging logic between Hadoop's abstract filesystem and Azure Storage + * Provides the bridging logic between Hadoop's abstract filesystem and Azure Storage * */ @InterfaceAudience.Private @VisibleForTesting public class AzureNativeFileSystemStore implements NativeFileSystemStore { - + /** * Configuration knob on whether we do block-level MD5 validation on * upload/download. @@ -102,6 +107,12 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { static final String DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME = "storageemulator"; static final String STORAGE_EMULATOR_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.storage.emulator.account.name"; + /** + * Configuration for User-Agent field. + */ + static final String USER_AGENT_ID_KEY = "fs.azure.user.agent.prefix"; + static final String USER_AGENT_ID_DEFAULT = "unknown"; + public static final Logger LOG = LoggerFactory.getLogger(AzureNativeFileSystemStore.class); private StorageInterface storageInteractionLayer; @@ -133,15 +144,15 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { private static final String KEY_MAX_BACKOFF_INTERVAL = "fs.azure.io.retry.max.backoff.interval"; private static final String KEY_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval"; private static final String KEY_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries"; - - private static final String KEY_COPYBLOB_MIN_BACKOFF_INTERVAL = + + private static final String KEY_COPYBLOB_MIN_BACKOFF_INTERVAL = "fs.azure.io.copyblob.retry.min.backoff.interval"; - private static final String KEY_COPYBLOB_MAX_BACKOFF_INTERVAL = + private static final String KEY_COPYBLOB_MAX_BACKOFF_INTERVAL = "fs.azure.io.copyblob.retry.max.backoff.interval"; - private static final String KEY_COPYBLOB_BACKOFF_INTERVAL = + private static final String KEY_COPYBLOB_BACKOFF_INTERVAL = "fs.azure.io.copyblob.retry.backoff.interval"; - private static final String KEY_COPYBLOB_MAX_IO_RETRIES = - "fs.azure.io.copyblob.retry.max.retries"; + private static final String KEY_COPYBLOB_MAX_IO_RETRIES = + "fs.azure.io.copyblob.retry.max.retries"; private static final String KEY_SELF_THROTTLE_ENABLE = "fs.azure.selfthrottling.enable"; private static final String KEY_SELF_THROTTLE_READ_FACTOR = "fs.azure.selfthrottling.read.factor"; @@ -188,7 +199,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * The set of directories where we should store files as page blobs. */ private Set<String> pageBlobDirs; - + /** * Configuration key to indicate the set of directories in WASB where * we should do atomic folder rename synchronized with createNonRecursive. @@ -232,11 +243,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { private static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s private static final int DEFAULT_BACKOFF_INTERVAL = 1 * 1000; // 1s private static final int DEFAULT_MAX_RETRY_ATTEMPTS = 15; - + private static final int DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL = 3 * 1000; private static final int DEFAULT_COPYBLOB_MAX_BACKOFF_INTERVAL = 90 * 1000; private static final int DEFAULT_COPYBLOB_BACKOFF_INTERVAL = 30 * 1000; - private static final int DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS = 15; + private static final int DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS = 15; // Self-throttling defaults. Allowed range = (0,1.0] // Value of 1.0 means no self-throttling. @@ -306,6 +317,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { private boolean useSecureMode = false; private boolean useLocalSasKeyMode = false; + // User-Agent + private String userAgentId; + private String delegationToken; /** The error message template when container is not accessible. */ @@ -319,7 +333,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * A test hook interface that can modify the operation context we use for * Azure Storage operations, e.g. to inject errors. */ - @VisibleForTesting + @VisibleForTesting interface TestHookOperationContext { OperationContext modifyOperationContext(OperationContext original); } @@ -336,11 +350,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Add a test hook to modify the operation context we use for Azure Storage * operations. - * + * * @param testHook * The test hook, or null to unset previous hooks. */ - @VisibleForTesting + @VisibleForTesting void addTestHookToOperationContext(TestHookOperationContext testHook) { this.testHookOperationContext = testHook; } @@ -358,7 +372,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Creates a JSON serializer that can serialize a PermissionStatus object into * the JSON string we want in the blob metadata. - * + * * @return The JSON serializer. */ private static JSON createPermissionJsonSerializer() { @@ -425,7 +439,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Check if concurrent reads and writes on the same blob are allowed. - * + * * @return true if concurrent reads and OOB writes has been configured, false * otherwise. */ @@ -437,11 +451,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * Method for the URI and configuration object necessary to create a storage * session with an Azure session. It parses the scheme to ensure it matches * the storage protocol supported by this file system. - * + * * @param uri - URI for target storage blob. * @param conf - reference to configuration object. * @param instrumentation - the metrics source that will keep track of operations here. - * + * * @throws IllegalArgumentException if URI or job object is null, or invalid scheme. */ @Override @@ -504,6 +518,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { pageBlobDirs = getDirectorySet(KEY_PAGE_BLOB_DIRECTORIES); LOG.debug("Page blob directories: {}", setToString(pageBlobDirs)); + // User-agent + userAgentId = conf.get(USER_AGENT_ID_KEY, USER_AGENT_ID_DEFAULT); + // Extract directories that should have atomic rename applied. atomicRenameDirs = getDirectorySet(KEY_ATOMIC_RENAME_DIRECTORIES); String hbaseRoot; @@ -539,7 +556,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Method to extract the account name from an Azure URI. - * + * * @param uri * -- WASB blob URI * @returns accountName -- the account name for the URI. @@ -590,7 +607,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Method to extract the container name from an Azure URI. - * + * * @param uri * -- WASB blob URI * @returns containerName -- the container name for the URI. May be null. @@ -641,7 +658,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Get the appropriate return the appropriate scheme for communicating with * Azure depending on whether wasb or wasbs is specified in the target URI. - * + * * return scheme - HTTPS or HTTP as appropriate. */ private String getHTTPScheme() { @@ -663,7 +680,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Set the configuration parameters for this client storage session with * Azure. - * + * * @throws AzureException */ private void configureAzureStorageSession() throws AzureException { @@ -763,10 +780,10 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Connect to Azure storage using anonymous credentials. - * + * * @param uri * - URI to target blob (R/O access to public blob) - * + * * @throws StorageException * raised on errors communicating with Azure storage. * @throws IOException @@ -893,7 +910,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { STORAGE_EMULATOR_ACCOUNT_NAME_PROPERTY_NAME, DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME)); } - + @VisibleForTesting public static String getAccountKeyFromConfiguration(String accountName, Configuration conf) throws KeyProviderException { @@ -930,7 +947,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * Establish a session with Azure blob storage based on the target URI. The * method determines whether or not the URI target contains an explicit * account or an implicit default cluster-wide account. - * + * * @throws AzureException * @throws IOException */ @@ -983,7 +1000,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { instrumentation.setAccountName(accountName); String containerName = getContainerFromAuthority(sessionUri); instrumentation.setContainerName(containerName); - + // Check whether this is a storage emulator account. if (isStorageEmulatorAccount(accountName)) { // It is an emulator account, connect to it with no credentials. @@ -1086,7 +1103,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { */ private String verifyAndConvertToStandardFormat(String rawDir) throws URISyntaxException { URI asUri = new URI(rawDir); - if (asUri.getAuthority() == null + if (asUri.getAuthority() == null || asUri.getAuthority().toLowerCase(Locale.ENGLISH).equalsIgnoreCase( sessionUri.getAuthority().toLowerCase(Locale.ENGLISH))) { // Applies to me. @@ -1167,8 +1184,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { return false; } - - + + /** * This should be called from any method that does any modifications to the * underlying container: it makes sure to put the WASB current version in the @@ -1364,11 +1381,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * could mean either: * (1) container=mycontainer; blob=myblob.txt * (2) container=$root; blob=mycontainer/myblob.txt - * + * * To avoid this type of ambiguity the Azure blob storage prevents * arbitrary path under $root. For a simple and more consistent user * experience it was decided to eliminate the opportunity for creating - * such paths by making the $root container read-only under WASB. + * such paths by making the $root container read-only under WASB. */ // Check that no attempt is made to write to blobs on default @@ -1445,7 +1462,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Default permission to use when no permission metadata is found. - * + * * @return The default permission to use. */ private static PermissionStatus defaultPermissionNoBlobMetadata() { @@ -1688,7 +1705,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Private method to check for authenticated access. - * + * * @ returns boolean -- true if access is credentialed and authenticated and * false otherwise. */ @@ -1708,7 +1725,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * original file system object was constructed with a short- or long-form URI. * If the root directory is non-null the URI in the file constructor was in * the long form. - * + * * @param includeMetadata * if set, the listed items will have their metadata populated * already. @@ -1717,7 +1734,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * * @returns blobItems : iterable collection of blob items. * @throws URISyntaxException - * + * */ private Iterable<ListBlobItem> listRootBlobs(boolean includeMetadata, boolean useFlatBlobListing) throws StorageException, URISyntaxException { @@ -1736,7 +1753,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * the directory depending on whether the original file system object was * constructed with a short- or long-form URI. If the root directory is * non-null the URI in the file constructor was in the long form. - * + * * @param aPrefix * : string name representing the prefix of containing blobs. * @param includeMetadata @@ -1744,10 +1761,10 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * already. * @param useFlatBlobListing * if set the list is flat, otherwise it is hierarchical. - * + * * @returns blobItems : iterable collection of blob items. * @throws URISyntaxException - * + * */ private Iterable<ListBlobItem> listRootBlobs(String aPrefix, boolean includeMetadata, boolean useFlatBlobListing) throws StorageException, URISyntaxException { @@ -1769,7 +1786,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * constructed with a short- or long-form URI. It also uses the specified flat * or hierarchical option, listing details options, request options, and * operation context. - * + * * @param aPrefix * string name representing the prefix of containing blobs. * @param useFlatBlobListing @@ -1784,7 +1801,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * - context of the current operation * @returns blobItems : iterable collection of blob items. * @throws URISyntaxException - * + * */ private Iterable<ListBlobItem> listRootBlobs(String aPrefix, boolean useFlatBlobListing, EnumSet<BlobListingDetails> listingDetails, BlobRequestOptions options, @@ -1804,13 +1821,13 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * get the block blob reference depending on whether the original file system * object was constructed with a short- or long-form URI. If the root * directory is non-null the URI in the file constructor was in the long form. - * + * * @param aKey * : a key used to query Azure for the block blob. * @returns blob : a reference to the Azure block blob corresponding to the * key. * @throws URISyntaxException - * + * */ private CloudBlobWrapper getBlobReference(String aKey) throws StorageException, URISyntaxException { @@ -1831,10 +1848,10 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * This private method normalizes the key by stripping the container name from * the path and returns a path relative to the root directory of the * container. - * + * * @param keyUri * - adjust this key to a path relative to the root directory - * + * * @returns normKey */ private String normalizeKey(URI keyUri) { @@ -1853,11 +1870,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * This private method normalizes the key by stripping the container name from * the path and returns a path relative to the root directory of the * container. - * + * * @param blob * - adjust the key to this blob to a path relative to the root * directory - * + * * @returns normKey */ private String normalizeKey(CloudBlobWrapper blob) { @@ -1868,11 +1885,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * This private method normalizes the key by stripping the container name from * the path and returns a path relative to the root directory of the * container. - * + * * @param directory * - adjust the key to this directory to a path relative to the root * directory - * + * * @returns normKey */ private String normalizeKey(CloudBlobDirectoryWrapper directory) { @@ -1889,7 +1906,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * operation that has listeners hooked to it that will update the metrics for * this file system. This method does not bind to receive send request * callbacks by default. - * + * * @return The OperationContext object to use. */ private OperationContext getInstrumentedContext() { @@ -1900,16 +1917,27 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Creates a new OperationContext for the Azure Storage operation that has * listeners hooked to it that will update the metrics for this file system. - * + * * @param bindConcurrentOOBIo * - bind to intercept send request call backs to handle OOB I/O. - * + * * @return The OperationContext object to use. */ private OperationContext getInstrumentedContext(boolean bindConcurrentOOBIo) { OperationContext operationContext = new OperationContext(); + // Set User-Agent + operationContext.getSendingRequestEventHandler().addListener(new StorageEvent<SendingRequestEvent>() { + @Override + public void eventOccurred(SendingRequestEvent eventArg) { + HttpURLConnection connection = (HttpURLConnection) eventArg.getConnectionObject(); + String userAgentInfo = String.format(Utility.LOCALE_US, "WASB/%s (%s) %s", + VersionInfo.getVersion(), userAgentId, BaseRequest.getUserAgent()); + connection.setRequestProperty(Constants.HeaderConstants.USER_AGENT, userAgentInfo); + } + }); + if (selfThrottlingEnabled) { SelfThrottlingIntercept.hook(operationContext, selfThrottlingReadFactor, selfThrottlingWriteFactor); @@ -2096,7 +2124,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Searches the given list of {@link FileMetadata} objects for a directory * with the given key. - * + * * @param list * The list to search. * @param key @@ -2229,7 +2257,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * Build up a metadata list of blobs in an Azure blob directory. This method * uses a in-order first traversal of blob directory structures to maintain * the sorted order of the blob names. - * + * * @param aCloudBlobDirectory Azure blob directory * @param aFileMetadataList a list of file metadata objects for each * non-directory blob. @@ -2564,7 +2592,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { // // Copy blob operation in Azure storage is very costly. It will be highly // likely throttled during Azure storage gc. Short term fix will be using - // a more intensive exponential retry policy when the cluster is getting + // a more intensive exponential retry policy when the cluster is getting // throttled. try { dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext()); @@ -2585,10 +2613,10 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { int copyBlobMaxRetries = sessionConfiguration.getInt( KEY_COPYBLOB_MAX_IO_RETRIES, DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS); - + BlobRequestOptions options = new BlobRequestOptions(); options.setRetryPolicyFactory(new RetryExponentialRetry( - copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff, + copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff, copyBlobMaxRetries)); dstBlob.startCopyFromBlob(srcBlob, options, getInstrumentedContext()); } else { @@ -2794,7 +2822,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { bandwidthGaugeUpdater = null; } } - + // Finalizer to ensure complete shutdown @Override protected void finalize() throws Throwable { http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2921e51/hadoop-tools/hadoop-azure/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md index 7415e29..79cb0ea 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md @@ -192,6 +192,19 @@ The configuration option `fs.azure.page.blob.extension.size` is the page blob extension size. This defines the amount to extend a page blob if it starts to get full. It must be 128MB or greater, specified as an integer number of bytes. +### Custom User-Agent +WASB passes User-Agent header to the Azure back-end. The default value +contains WASB version, Java Runtime version, Azure Client library version, and the +value of the configuration option `fs.azure.user.agent.prefix`. Customized User-Agent +header enables better troubleshooting and analysis by Azure service. + +```xml +<property> + <name>fs.azure.user.agent.prefix</name> + <value>Identifier</value> +</property> +``` + ### Atomic Folder Rename Azure storage stores files as a flat key/value store without formal support http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2921e51/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java index 194a831..672ed9c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java @@ -566,4 +566,52 @@ public class TestWasbUriAndConfiguration { CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, null); assertEquals(newPath, effectivePath); } + + @Test + public void testUserAgentConfig() throws Exception { + // Set the user agent + try { + testAccount = AzureBlobStorageTestAccount.createMock(); + Configuration conf = testAccount.getFileSystem().getConf(); + String authority = testAccount.getFileSystem().getUri().getAuthority(); + URI defaultUri = new URI("wasbs", authority, null, null, null); + conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); + conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs"); + + conf.set(AzureNativeFileSystemStore.USER_AGENT_ID_KEY, "TestClient"); + + FileSystem fs = FileSystem.get(conf); + AbstractFileSystem afs = FileContext.getFileContext(conf).getDefaultFileSystem(); + + assertTrue(afs instanceof Wasbs); + assertEquals(-1, afs.getUri().getPort()); + assertEquals("wasbs", afs.getUri().getScheme()); + + } finally { + testAccount.cleanup(); + FileSystem.closeAll(); + } + + // Unset the user agent + try { + testAccount = AzureBlobStorageTestAccount.createMock(); + Configuration conf = testAccount.getFileSystem().getConf(); + String authority = testAccount.getFileSystem().getUri().getAuthority(); + URI defaultUri = new URI("wasbs", authority, null, null, null); + conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); + conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs"); + + conf.unset(AzureNativeFileSystemStore.USER_AGENT_ID_KEY); + + FileSystem fs = FileSystem.get(conf); + AbstractFileSystem afs = FileContext.getFileContext(conf).getDefaultFileSystem(); + assertTrue(afs instanceof Wasbs); + assertEquals(-1, afs.getUri().getPort()); + assertEquals("wasbs", afs.getUri().getScheme()); + + } finally { + testAccount.cleanup(); + FileSystem.closeAll(); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2921e51/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml index acd9459..8c88743 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml @@ -34,6 +34,11 @@ <value>true</value> </property> + <property> + <name>fs.azure.user.agent.prefix</name> + <value>MSFT</value> + </property> + <!-- Save the above configuration properties in a separate file named --> <!-- azure-auth-keys.xml in the same directory as this file. --> <!-- DO NOT ADD azure-auth-keys.xml TO REVISION CONTROL. The keys to your --> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
