Repository: hadoop Updated Branches: refs/heads/branch-2.7 01f5e609a -> 7595d8961
HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri. Cherry picked from commit #8652cce5b2 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7595d896 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7595d896 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7595d896 Branch: refs/heads/branch-2.7 Commit: 7595d89612f4916ab29a6ac4e07374771fcbc94f Parents: 01f5e60 Author: Chris Douglas <[email protected]> Authored: Fri Dec 18 18:21:52 2015 -0800 Committer: Steve Loughran <[email protected]> Committed: Mon Jun 27 14:32:42 2016 +0100 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/FileSystem.java | 18 ++++++++++++++++-- .../hadoop/fs/TestFileSystemInitialization.java | 16 ++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7595d896/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index fac3c40..2e684f5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -35,6 +35,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.ServiceConfigurationError; import java.util.ServiceLoader; import java.util.Set; import java.util.Stack; @@ -61,6 +62,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ClassUtil; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; @@ -2622,8 +2624,20 @@ public abstract class FileSystem extends Configured implements Closeable { synchronized (FileSystem.class) { if (!FILE_SYSTEMS_LOADED) { ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class); - for (FileSystem fs : serviceLoader) { - SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass()); + Iterator<FileSystem> it = serviceLoader.iterator(); + while (it.hasNext()) { + FileSystem fs = null; + try { + fs = it.next(); + try { + SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass()); + } catch (Exception e) { + LOG.warn("Cannot load: " + fs + " from " + + ClassUtil.findContainingJar(fs.getClass()), e); + } + } catch (ServiceConfigurationError ee) { + LOG.warn("Cannot load filesystem", ee); + } } FILE_SYSTEMS_LOADED = true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7595d896/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java index d3fceec..18e8b01 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import java.io.IOException; import java.net.URL; +import java.util.ServiceConfigurationError; import org.junit.Test; import static org.junit.Assert.*; @@ -43,4 +44,19 @@ public class TestFileSystemInitialization { assertFalse(false); } } + + @Test + public void testMissingLibraries() { + boolean catched = false; + try { + Configuration conf = new Configuration(); + FileSystem.getFileSystemClass("s3a", conf); + } catch (Exception e) { + catched = true; + } catch (ServiceConfigurationError e) { + // S3A shouldn't find AWS SDK and fail + catched = true; + } + assertTrue(catched); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
