This is an automated email from the ASF dual-hosted git repository. shv pushed a commit to branch branch-2.10 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push: new 99b952f HADOOP-17028. ViewFS should initialize mounted target filesystems lazily. Contributed by Abhishek Das (#2260, #3218) 99b952f is described below commit 99b952fcb49640aab05a72108653845968a611d1 Author: Abhishek Das <abhishek.b...@gmail.com> AuthorDate: Tue Jul 13 12:47:43 2021 -0700 HADOOP-17028. ViewFS should initialize mounted target filesystems lazily. Contributed by Abhishek Das (#2260, #3218) (cherry picked from commit 1dd03cc4b573270dc960117c3b6c74bb78215caa) --- .../org/apache/hadoop/fs/viewfs/InodeTree.java | 51 ++++++++++--- .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 85 +++++++++++++++++----- .../java/org/apache/hadoop/fs/viewfs/ViewFs.java | 42 ++++++++--- .../apache/hadoop/fs/viewfs/TestViewFsConfig.java | 7 +- .../hadoop/fs/viewfs/ViewFileSystemBaseTest.java | 44 ++++++++++- .../hadoop/fs/viewfs/TestViewFileSystemHdfs.java | 76 +++++++++++++++++++ .../apache/hadoop/fs/viewfs/TestViewFsHdfs.java | 78 ++++++++++++++++++++ 7 files changed, 338 insertions(+), 45 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index 0a949fe..779cec8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.viewfs; +import com.google.common.base.Function; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; @@ -158,8 +159,10 @@ abstract class InodeTree<T> { static class INodeLink<T> extends INode<T> { final boolean isMergeLink; // true if MergeLink final URI[] targetDirLinkList; - final T targetFileSystem; // file system object created from the link. - + private T targetFileSystem; // file system object created from the link. + // Function to initialize file system. Only applicable for simple links + private Function<URI, T> fileSystemInitMethod; + private final Object lock = new Object(); /** * Construct a mergeLink */ @@ -175,12 +178,14 @@ abstract class InodeTree<T> { * Construct a simple link (i.e. not a mergeLink) */ INodeLink(final String pathToNode, final UserGroupInformation aUgi, - final T targetFs, final URI aTargetDirLink) { + Function<URI, T> createFileSystemMethod, + final URI aTargetDirLink) { super(pathToNode, aUgi); - targetFileSystem = targetFs; + targetFileSystem = null; targetDirLinkList = new URI[1]; targetDirLinkList[0] = aTargetDirLink; isMergeLink = false; + this.fileSystemInitMethod = createFileSystemMethod; } /** @@ -196,6 +201,33 @@ abstract class InodeTree<T> { } return new Path(result.toString()); } + + /** + * Get the instance of FileSystem to use, creating one if needed. + * @return An Initialized instance of T + * @throws IOException + */ + public T getTargetFileSystem() throws IOException { + if (targetFileSystem != null) { + return targetFileSystem; + } + // For non NFLY and MERGE links, we initialize the FileSystem when the + // corresponding mount path is accessed. + if (targetDirLinkList.length == 1) { + synchronized (lock) { + if (targetFileSystem != null) { + return targetFileSystem; + } + targetFileSystem = fileSystemInitMethod.apply(targetDirLinkList[0]); + if (targetFileSystem == null) { + throw new IOException( + "Could not initialize target File System for URI : " + + targetDirLinkList[0]); + } + } + } + return targetFileSystem; + } } @@ -258,7 +290,7 @@ abstract class InodeTree<T> { getTargetFileSystem(targetsListURI), targetsListURI); } else { newLink = new INodeLink<T>(fullPath, aUgi, - getTargetFileSystem(new URI(target)), new URI(target)); + initAndGetTargetFs(), new URI(target)); } curInode.addLink(iPath, newLink); mountPoints.add(new MountPoint<T>(src, newLink)); @@ -267,14 +299,13 @@ abstract class InodeTree<T> { /** * Below the "public" methods of InodeTree */ - + /** * The user of this class must subclass and implement the following * 3 abstract methods. * @throws IOException */ - protected abstract T getTargetFileSystem(final URI uri) - throws UnsupportedFileSystemException, URISyntaxException, IOException; + protected abstract Function<URI, T> initAndGetTargetFs(); protected abstract T getTargetFileSystem(final INodeDir<T> dir) throws URISyntaxException; @@ -385,7 +416,7 @@ abstract class InodeTree<T> { * @throws FileNotFoundException */ ResolveResult<T> resolve(final String p, final boolean resolveLastComponent) - throws FileNotFoundException { + throws IOException { // TO DO: - more efficient to not split the path, but simply compare String[] path = breakIntoPathComponents(p); if (path.length <= 1) { // special case for when path is "/" @@ -422,7 +453,7 @@ abstract class InodeTree<T> { } final ResolveResult<T> res = new ResolveResult<T>(ResultKind.isExternalDir, - link.targetFileSystem, nextInode.fullPath, remainingPath); + link.getTargetFileSystem(), nextInode.fullPath, remainingPath); return res; } else if (nextInode instanceof INodeDir) { curInode = (INodeDir<T>) nextInode; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 7960048..5ee706d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -21,10 +21,12 @@ import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT; +import com.google.common.base.Function; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; @@ -237,7 +239,7 @@ public class ViewFileSystem extends FileSystem { config = conf; enableInnerCache = config.getBoolean(CONFIG_VIEWFS_ENABLE_INNER_CACHE, CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT); - final InnerCache innerCache = new InnerCache(); + cache = new InnerCache(); // Now build client side view (i.e. client side mount table) from config. final String authority = theUri.getAuthority(); try { @@ -245,16 +247,32 @@ public class ViewFileSystem extends FileSystem { fsState = new InodeTree<FileSystem>(conf, authority) { @Override - protected - FileSystem getTargetFileSystem(final URI uri) - throws URISyntaxException, IOException { - FileSystem fs; - if (enableInnerCache) { - fs = innerCache.get(uri, config); - } else { - fs = FileSystem.get(uri, config); + protected Function<URI, FileSystem> initAndGetTargetFs() { + return new Function<URI, FileSystem>() { + @Override + public FileSystem apply(final URI uri) { + FileSystem fs; + try { + fs = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { + @Override + public FileSystem run() throws IOException { + if (enableInnerCache) { + synchronized (cache) { + return cache.get(uri, config); + } + } else { + return FileSystem.get(uri, config); + } + } + }); + return new ChRootedFileSystem(fs, uri); + } catch (IOException | InterruptedException ex) { + LOG.error("Could not initialize the underlying FileSystem " + + "object. Exception: " + ex.toString()); + } + return null; } - return new ChRootedFileSystem(fs, uri); + }; } @Override @@ -273,12 +291,6 @@ public class ViewFileSystem extends FileSystem { } }; - if (enableInnerCache) { - // All fs instances are created and cached on startup. The cache is - // readonly after the initialize() so the concurrent access of the cache - // is safe. - cache = innerCache.unmodifiableCache(); - } workingDir = this.getHomeDirectory(); renameStrategy = RenameStrategy.valueOf( conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, @@ -311,7 +323,7 @@ public class ViewFileSystem extends FileSystem { this(FsConstants.VIEWFS_URI, conf); } - public Path getTrashCanLocation(final Path f) throws FileNotFoundException { + public Path getTrashCanLocation(final Path f) throws IOException { final InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(f), true); return res.isInternalDir() ? null : res.targetFileSystem.getHomeDirectory(); @@ -767,9 +779,34 @@ public class ViewFileSystem extends FileSystem { public void setVerifyChecksum(final boolean verifyChecksum) { List<InodeTree.MountPoint<FileSystem>> mountPoints = fsState.getMountPoints(); + Map<String, FileSystem> fsMap = initializeMountedFileSystems(mountPoints); + for (InodeTree.MountPoint<FileSystem> mount : mountPoints) { + fsMap.get(mount.src).setVerifyChecksum(verifyChecksum); + } + } + + /** + * Initialize the target filesystem for all mount points. + * @param mountPoints The mount points + * @return Mapping of mount point and the initialized target filesystems + * @throws RuntimeException when the target file system cannot be initialized + */ + private Map<String, FileSystem> initializeMountedFileSystems( + List<InodeTree.MountPoint<FileSystem>> mountPoints) { + FileSystem fs = null; + Map<String, FileSystem> fsMap = new HashMap<>(mountPoints.size()); for (InodeTree.MountPoint<FileSystem> mount : mountPoints) { - mount.target.targetFileSystem.setVerifyChecksum(verifyChecksum); + try { + fs = mount.target.getTargetFileSystem(); + fsMap.put(mount.src, fs); + } catch (IOException ex) { + String errMsg = "Not able to initialize FileSystem for mount path " + + mount.src + " with exception " + ex; + LOG.error(errMsg); + throw new RuntimeException(errMsg, ex); + } } + return fsMap; } @Override @@ -795,6 +832,9 @@ public class ViewFileSystem extends FileSystem { return res.targetFileSystem.getDefaultBlockSize(res.remainingPath); } catch (FileNotFoundException e) { throw new NotInMountpointException(f, "getDefaultBlockSize"); + } catch (IOException e) { + throw new RuntimeException("Not able to initialize fs in " + + " getDefaultBlockSize for path " + f + " with exception", e); } } @@ -806,6 +846,9 @@ public class ViewFileSystem extends FileSystem { return res.targetFileSystem.getDefaultReplication(res.remainingPath); } catch (FileNotFoundException e) { throw new NotInMountpointException(f, "getDefaultReplication"); + } catch (IOException e) { + throw new RuntimeException("Not able to initialize fs in " + + " getDefaultReplication for path " + f + " with exception", e); } } @@ -834,8 +877,9 @@ public class ViewFileSystem extends FileSystem { public void setWriteChecksum(final boolean writeChecksum) { List<InodeTree.MountPoint<FileSystem>> mountPoints = fsState.getMountPoints(); + Map<String, FileSystem> fsMap = initializeMountedFileSystems(mountPoints); for (InodeTree.MountPoint<FileSystem> mount : mountPoints) { - mount.target.targetFileSystem.setWriteChecksum(writeChecksum); + fsMap.get(mount.src).setWriteChecksum(writeChecksum); } } @@ -843,9 +887,10 @@ public class ViewFileSystem extends FileSystem { public FileSystem[] getChildFileSystems() { List<InodeTree.MountPoint<FileSystem>> mountPoints = fsState.getMountPoints(); + Map<String, FileSystem> fsMap = initializeMountedFileSystems(mountPoints); Set<FileSystem> children = new HashSet<FileSystem>(); for (InodeTree.MountPoint<FileSystem> mountPoint : mountPoints) { - FileSystem targetFs = mountPoint.target.targetFileSystem; + FileSystem targetFs = fsMap.get(mountPoint.src); children.addAll(Arrays.asList(targetFs.getChildFileSystems())); } return children.toArray(new FileSystem[]{}); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 364485f..1440a4b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -19,10 +19,12 @@ package org.apache.hadoop.fs.viewfs; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import com.google.common.base.Function; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -65,6 +67,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -152,6 +156,7 @@ import org.apache.hadoop.util.Time; @InterfaceAudience.Public @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */ public class ViewFs extends AbstractFileSystem { + static final Logger LOG = LoggerFactory.getLogger(ViewFs.class); final long creationTime; // of the the mount table final UserGroupInformation ugi; // the user/group of user who created mtable final Configuration config; @@ -212,16 +217,32 @@ public class ViewFs extends AbstractFileSystem { fsState = new InodeTree<AbstractFileSystem>(conf, authority) { @Override - protected - AbstractFileSystem getTargetFileSystem(final URI uri) - throws URISyntaxException, UnsupportedFileSystemException { - String pathString = uri.getPath(); - if (pathString.isEmpty()) { - pathString = "/"; + protected Function<URI, AbstractFileSystem> initAndGetTargetFs() { + return new Function<URI, AbstractFileSystem>() { + @Override + public AbstractFileSystem apply(final URI uri) { + AbstractFileSystem fs; + try { + fs = ugi.doAs( + new PrivilegedExceptionAction<AbstractFileSystem>() { + @Override + public AbstractFileSystem run() throws IOException { + return AbstractFileSystem.createFileSystem(uri, config); + } + }); + String pathString = uri.getPath(); + if (pathString.isEmpty()) { + pathString = "/"; + } + return new ChRootedFs(fs, new Path(pathString)); + } catch (IOException | URISyntaxException | + InterruptedException ex) { + LOG.error("Could not initialize underlying FileSystem object" + +" for uri " + uri + "with exception: " + ex.toString()); + } + return null; } - return new ChRootedFs( - AbstractFileSystem.createFileSystem(uri, config), - new Path(pathString)); + }; } @Override @@ -624,7 +645,8 @@ public class ViewFs extends AbstractFileSystem { List<Token<?>> result = new ArrayList<Token<?>>(initialListSize); for ( int i = 0; i < mountPoints.size(); ++i ) { List<Token<?>> tokens = - mountPoints.get(i).target.targetFileSystem.getDelegationTokens(renewer); + mountPoints.get(i).target.getTargetFileSystem() + .getDelegationTokens(renewer); if (tokens != null) { result.addAll(tokens); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java index 75b329c..daf68c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.viewfs; +import com.google.common.base.Function; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -47,10 +48,8 @@ public class TestViewFsConfig { new InodeTree<Foo>(conf, null) { @Override - protected - Foo getTargetFileSystem(final URI uri) - throws URISyntaxException, UnsupportedFileSystemException { - return null; + protected Function<URI, Foo> initAndGetTargetFs() { + return null; } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 025fe67..d24d92a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileSystemTestHelper; import static org.apache.hadoop.fs.FileSystemTestHelper.*; import org.apache.hadoop.fs.permission.AclEntry; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.test.GenericTestUtils.*; import static org.junit.Assert.*; @@ -65,7 +66,6 @@ import org.junit.Before; import org.junit.Test; import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.*; /** * <p> @@ -1037,6 +1037,8 @@ abstract public class ViewFileSystemBaseTest { final int cacheSize = TestFileUtil.getCacheSize(); ViewFileSystem viewFs = (ViewFileSystem) FileSystem .get(new URI("viewfs://" + clusterName + "/"), config); + viewFs.resolvePath( + new Path(String.format("viewfs://%s/%s", clusterName, "/user"))); assertEquals(cacheSize + 1, TestFileUtil.getCacheSize()); viewFs.close(); assertEquals(cacheSize, TestFileUtil.getCacheSize()); @@ -1087,4 +1089,44 @@ abstract public class ViewFileSystemBaseTest { } } + @Test + public void testTargetFileSystemLazyInitialization() throws Exception { + final String clusterName = "cluster" + new Random().nextInt(); + Configuration config = new Configuration(conf); + config.setBoolean(CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); + config.setClass("fs.mockfs.impl", + TestChRootedFileSystem.MockFileSystem.class, FileSystem.class); + ConfigUtil.addLink(config, clusterName, "/user", + URI.create("mockfs://mockauth1/mockpath")); + ConfigUtil.addLink(config, clusterName, + "/mock", URI.create("mockfs://mockauth/mockpath")); + + final int cacheSize = TestFileUtil.getCacheSize(); + ViewFileSystem viewFs = (ViewFileSystem) FileSystem + .get(new URI("viewfs://" + clusterName + "/"), config); + + // As no inner file system instance has been initialized, + // cache size will remain the same + // cache is disabled for viewfs scheme, so the viewfs:// instance won't + // go in the cache even after the initialization + assertEquals(cacheSize, TestFileUtil.getCacheSize()); + + // This resolve path will initialize the file system corresponding + // to the mount table entry of the path "/user" + viewFs.resolvePath( + new Path(String.format("viewfs://%s/%s", clusterName, "/user"))); + + // Cache size will increase by 1. + assertEquals(cacheSize + 1, TestFileUtil.getCacheSize()); + // This resolve path will initialize the file system corresponding + // to the mount table entry of the path "/mock" + viewFs.resolvePath(new Path(String.format("viewfs://%s/%s", clusterName, + "/mock"))); + // One more file system instance will get initialized. + assertEquals(cacheSize + 2, TestFileUtil.getCacheSize()); + viewFs.close(); + // Initialized FileSystem instances will not be removed from cache as + // viewfs inner cache is disabled + assertEquals(cacheSize + 2, TestFileUtil.getCacheSize()); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java index 7a31a37..b8da0c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java @@ -20,7 +20,10 @@ package org.apache.hadoop.fs.viewfs; import java.io.IOException; import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; +import java.util.HashMap; +import java.util.Map; import javax.security.auth.login.LoginException; import org.apache.hadoop.conf.Configuration; @@ -30,12 +33,17 @@ import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; + +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.junit.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { @@ -155,4 +163,72 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { e); } } + + @Test + public void testTargetFileSystemLazyInitializationWithUgi() throws Exception { + final Map<String, FileSystem> map = new HashMap<>(); + final Path user1Path = new Path("/data/user1"); + + // Scenario - 1: Create FileSystem with the current user context + // Both mkdir and delete should be successful + FileSystem fs = FileSystem.get(FsConstants.VIEWFS_URI, conf); + fs.mkdirs(user1Path); + fs.delete(user1Path, false); + + // Scenario - 2: Create FileSystem with the a different user context + final UserGroupInformation userUgi = UserGroupInformation + .createUserForTesting("us...@hadoop.com", new String[]{"hadoop"}); + userUgi.doAs(new PrivilegedExceptionAction<Object>() { + @Override + public Object run() throws IOException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + String doAsUserName = ugi.getUserName(); + assertEquals(doAsUserName, "us...@hadoop.com"); + + FileSystem viewFS = FileSystem.get(FsConstants.VIEWFS_URI, conf); + map.put("user1", viewFS); + return null; + } + }); + + // Try creating a directory with the file context created by a different ugi + // Though we are running the mkdir with the current user context, the + // target filesystem will be instantiated by the ugi with which the + // file context was created. + try { + FileSystem otherfs = map.get("user1"); + otherfs.mkdirs(user1Path); + fail("This mkdir should fail"); + } catch (AccessControlException ex) { + // Exception is expected as the FileSystem was created with ugi of user1 + // So when we are trying to access the /user/user1 path for the first + // time, the corresponding file system is initialized and it tries to + // execute the task with ugi with which the FileSystem was created. + } + + // Change the permission of /data path so that user1 can create a directory + fsTarget.setOwner(new Path(targetTestRoot, "data"), + "user1", "test2"); + // set permission of target to allow rename to target + fsTarget.setPermission(new Path(targetTestRoot, "data"), + new FsPermission("775")); + + userUgi.doAs(new PrivilegedExceptionAction<Object>() { + @Override + public Object run() throws IOException { + FileSystem viewFS = FileSystem.get(FsConstants.VIEWFS_URI, conf); + map.put("user1", viewFS); + return null; + } + }); + + // Although we are running with current user context, and current user + // context does not have write permission, we are able to create the + // directory as its using ugi of user1 which has write permission. + FileSystem otherfs = map.get("user1"); + otherfs.mkdirs(user1Path); + String owner = otherfs.getFileStatus(user1Path).getOwner(); + assertEquals("The owner did not match ", owner, userUgi.getShortUserName()); + otherfs.delete(user1Path, false); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java index fda6672..540883d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java @@ -21,18 +21,28 @@ package org.apache.hadoop.fs.viewfs; import java.io.IOException; import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; +import java.util.HashMap; +import java.util.Map; import javax.security.auth.login.LoginException; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContextTestHelper; +import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class TestViewFsHdfs extends ViewFsBaseTest { @@ -85,5 +95,73 @@ public class TestViewFsHdfs extends ViewFsBaseTest { int getExpectedDelegationTokenCount() { return 8; } + + @Test + public void testTargetFileSystemLazyInitialization() throws Exception { + final Map<String, FileContext> map = new HashMap<>(); + final Path user1Path = new Path("/data/user1"); + + // Scenario - 1: Create FileContext with the current user context + // Both mkdir and delete should be successful + FileContext fs = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf); + fs.mkdir(user1Path, FileContext.DEFAULT_PERM, false); + fs.delete(user1Path, false); + + // Scenario - 2: Create FileContext with the a different user context + final UserGroupInformation userUgi = UserGroupInformation + .createUserForTesting("us...@hadoop.com", new String[]{"hadoop"}); + userUgi.doAs(new PrivilegedExceptionAction<Object>() { + @Override + public Object run() throws IOException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + String doAsUserName = ugi.getUserName(); + assertEquals(doAsUserName, "us...@hadoop.com"); + + FileContext viewFS = FileContext.getFileContext( + FsConstants.VIEWFS_URI, conf); + map.put("user1", viewFS); + return null; + } + }); + + // Try creating a directory with the file context created by a different ugi + // Though we are running the mkdir with the current user context, the + // target filesystem will be instantiated by the ugi with which the + // file context was created. + try { + FileContext otherfs = map.get("user1"); + otherfs.mkdir(user1Path, FileContext.DEFAULT_PERM, false); + fail("This mkdir should fail"); + } catch (AccessControlException ex) { + // Exception is expected as the FileContext was created with ugi of user1 + // So when we are trying to access the /user/user1 path for the first + // time, the corresponding file system is initialized and it tries to + // execute the task with ugi with which the FileContext was created. + } + + // Change the permission of /data path so that user1 can create a directory + fcView.setOwner(new Path("/data"), "user1", "test2"); + // set permission of target to allow rename to target + fcView.setPermission(new Path("/data"), new FsPermission("775")); + + userUgi.doAs(new PrivilegedExceptionAction<Object>() { + @Override + public Object run() throws IOException { + FileContext viewFS = FileContext.getFileContext( + FsConstants.VIEWFS_URI, conf); + map.put("user1", viewFS); + return null; + } + }); + + // Although we are running with current user context, and current user + // context does not have write permission, we are able to create the + // directory as its using ugi of user1 which has write permission. + FileContext otherfs = map.get("user1"); + otherfs.mkdir(user1Path, FileContext.DEFAULT_PERM, false); + String owner = otherfs.getFileStatus(user1Path).getOwner(); + assertEquals("The owner did not match ", owner, userUgi.getShortUserName()); + otherfs.delete(user1Path, false); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org