HADOOP-12304. Applications using FileContext fail with the default file system configured to be wasb/s3/etc. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3da0beda Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3da0beda Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3da0beda Branch: refs/heads/HADOOP-12111 Commit: 3da0bedaec38714a62c3b7244e84bc4f981ddf45 Parents: ba2313d Author: cnauroth <[email protected]> Authored: Wed Aug 5 14:56:41 2015 -0700 Committer: cnauroth <[email protected]> Committed: Wed Aug 5 14:56:41 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../apache/hadoop/fs/DelegateToFileSystem.java | 19 +++++++++++-- .../fs/azure/TestWasbUriAndConfiguration.java | 30 ++++++++++++++++++-- .../src/test/resources/azure-test.xml | 4 +++ 4 files changed, 52 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3da0beda/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3951a99..7571cc5 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1076,6 +1076,9 @@ Release 2.7.2 - UNRELEASED HDFS-8767. RawLocalFileSystem.listStatus() returns null for UNIX pipefile. (kanaka kumar avvaru via wheat9) + HADOOP-12304. Applications using FileContext fail with the default file + system configured to be wasb/s3/etc. (cnauroth) + Release 2.7.1 - 2015-07-06 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/3da0beda/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java index 6b7f387..388e64b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java @@ -46,12 +46,27 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem { Configuration conf, String supportedScheme, boolean authorityRequired) throws IOException, URISyntaxException { super(theUri, supportedScheme, authorityRequired, - theFsImpl.getDefaultPort()); + getDefaultPortIfDefined(theFsImpl)); fsImpl = theFsImpl; fsImpl.initialize(theUri, conf); fsImpl.statistics = getStatistics(); } + /** + * Returns the default port if the file system defines one. + * {@link FileSystem#getDefaultPort()} returns 0 to indicate the default port + * is undefined. However, the logic that consumes this value expects to + * receive -1 to indicate the port is undefined, which agrees with the + * contract of {@link URI#getPort()}. + * + * @param theFsImpl file system to check for default port + * @return default port, or -1 if default port is undefined + */ + private static int getDefaultPortIfDefined(FileSystem theFsImpl) { + int defaultPort = theFsImpl.getDefaultPort(); + return defaultPort != 0 ? defaultPort : -1; + } + @Override public Path getInitialWorkingDirectory() { return fsImpl.getInitialWorkingDirectory(); @@ -239,4 +254,4 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem { public List<Token<?>> getDelegationTokens(String renewer) throws IOException { return Arrays.asList(fsImpl.addDelegationTokens(renewer, null)); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3da0beda/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java index a4ca6fd..25eee47 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.azure; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -35,6 +36,8 @@ import java.util.Date; import java.util.EnumSet; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions; @@ -362,7 +365,7 @@ public class TestWasbUriAndConfiguration { Configuration conf = testAccount.getFileSystem().getConf(); String authority = testAccount.getFileSystem().getUri().getAuthority(); URI defaultUri = new URI(defaultScheme, authority, null, null, null); - conf.set("fs.default.name", defaultUri.toString()); + conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); // Add references to file system implementations for wasb and wasbs. conf.addResource("azure-test.xml"); @@ -385,11 +388,34 @@ public class TestWasbUriAndConfiguration { // authority for the Azure file system should throw. testAccount = AzureBlobStorageTestAccount.createMock(); Configuration conf = testAccount.getFileSystem().getConf(); - conf.set("fs.default.name", "file:///"); + conf.set(FS_DEFAULT_NAME_KEY, "file:///"); try { FileSystem.get(new URI("wasb:///random/path"), conf); fail("Should've thrown."); } catch (IllegalArgumentException e) { } } + + @Test + public void testWasbAsDefaultFileSystemHasNoPort() throws Exception { + try { + testAccount = AzureBlobStorageTestAccount.createMock(); + Configuration conf = testAccount.getFileSystem().getConf(); + String authority = testAccount.getFileSystem().getUri().getAuthority(); + URI defaultUri = new URI("wasb", authority, null, null, null); + conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); + conf.addResource("azure-test.xml"); + + FileSystem fs = FileSystem.get(conf); + assertTrue(fs instanceof NativeAzureFileSystem); + assertEquals(-1, fs.getUri().getPort()); + + AbstractFileSystem afs = FileContext.getFileContext(conf) + .getDefaultFileSystem(); + assertTrue(afs instanceof Wasb); + assertEquals(-1, afs.getUri().getPort()); + } finally { + FileSystem.closeAll(); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3da0beda/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml index 98e68c4..75b466d 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml @@ -35,4 +35,8 @@ <value>true</value> </property> --> + <property> + <name>fs.AbstractFileSystem.wasb.impl</name> + <value>org.apache.hadoop.fs.azure.Wasb</value> + </property> </configuration>
