This is an automated email from the ASF dual-hosted git repository.

umamahesh pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
     new 5f67c3f  HDFS-15418. ViewFileSystemOverloadScheme should represent 
mount links as non symlinks. Contributed by Uma Maheswara Rao G.
5f67c3f is described below

commit 5f67c3f3ca49d718d8fc7c1914c3d2b77b3462f0
Author: Uma Maheswara Rao G <umamah...@apache.org>
AuthorDate: Sat Jun 20 00:32:02 2020 -0700

    HDFS-15418. ViewFileSystemOverloadScheme should represent mount links as 
non symlinks. Contributed by Uma Maheswara Rao G.
    
    (cherry picked from commit b27810aa6015253866ccc0ccc7247ad7024c0730)
---
 .../org/apache/hadoop/fs/viewfs/Constants.java     |   8 ++
 .../apache/hadoop/fs/viewfs/ViewFileSystem.java    |  71 +++++++----
 .../fs/viewfs/ViewFileSystemOverloadScheme.java    |  20 +++-
 .../java/org/apache/hadoop/fs/viewfs/ViewFs.java   |  80 ++++++++-----
 .../viewfs/TestViewFsOverloadSchemeListStatus.java | 132 +++++++++++++++++++++
 .../hadoop/fs/viewfs/TestViewfsFileStatus.java     |   4 +-
 .../src/site/markdown/ViewFsOverloadScheme.md      |  42 +++----
 ...SystemOverloadSchemeHdfsFileSystemContract.java |   5 +
 ...ViewFileSystemOverloadSchemeWithHdfsScheme.java |   9 ++
 9 files changed, 295 insertions(+), 76 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
index 0a5d4b4..f454f63 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
@@ -90,4 +90,12 @@ public interface Constants {
   String CONFIG_VIEWFS_ENABLE_INNER_CACHE = "fs.viewfs.enable.inner.cache";
 
   boolean CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT = true;
+
+  /**
+   * Enable ViewFileSystem to show mountlinks as symlinks.
+   */
+  String CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS =
+      "fs.viewfs.mount.links.as.symlinks";
+
+  boolean CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT = true;
 }
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 895edc0..1ee06e0 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.viewfs;
 import static 
org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
 import static 
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE;
 import static 
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT;
+import static 
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS;
+import static 
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT;
 import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
 
 import java.io.FileNotFoundException;
@@ -527,10 +529,18 @@ public class ViewFileSystem extends FileSystem {
    * the target path FileStatus object. The target path will be available via
    * getSymlink on that children's FileStatus object. Since it represents as
    * symlink, isDirectory on that children's FileStatus will return false.
+   * This behavior can be changed by setting an advanced configuration
+   * fs.viewfs.mount.links.as.symlinks to false. In this case, mount points 
will
+   * be represented as non-symlinks and all the file/directory attributes like
+   * permissions, isDirectory etc will be assigned from it's resolved target
+   * directory/file.
    *
    * If you want to get the FileStatus of target path for that children, you 
may
    * want to use GetFileStatus API with that children's symlink path. Please 
see
    * {@link ViewFileSystem#getFileStatus(Path f)}
+   *
+   * Note: In ViewFileSystem, by default the mount links are represented as
+   * symlinks.
    */
   @Override
   public FileStatus[] listStatus(final Path f) throws AccessControlException,
@@ -1114,6 +1124,7 @@ public class ViewFileSystem extends FileSystem {
     final long creationTime; // of the the mount table
     final UserGroupInformation ugi; // the user/group of user who created 
mtable
     final URI myUri;
+    private final boolean showMountLinksAsSymlinks;
     
     public InternalDirOfViewFs(final InodeTree.INodeDir<FileSystem> dir,
         final long cTime, final UserGroupInformation ugi, URI uri,
@@ -1127,6 +1138,9 @@ public class ViewFileSystem extends FileSystem {
       theInternalDir = dir;
       creationTime = cTime;
       this.ugi = ugi;
+      showMountLinksAsSymlinks = config
+          .getBoolean(CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS,
+              CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT);
     }
 
     static private void checkPathIsSlash(final Path f) throws IOException {
@@ -1216,37 +1230,50 @@ public class ViewFileSystem extends FileSystem {
       for (Entry<String, INode<FileSystem>> iEntry :
           theInternalDir.getChildren().entrySet()) {
         INode<FileSystem> inode = iEntry.getValue();
+        Path path = new Path(inode.fullPath).makeQualified(myUri, null);
         if (inode.isLink()) {
           INodeLink<FileSystem> link = (INodeLink<FileSystem>) inode;
+
+          if (showMountLinksAsSymlinks) {
+            // To maintain backward compatibility, with default option(showing
+            // mount links as symlinks), we will represent target link as
+            // symlink and rest other properties are belongs to mount link 
only.
+            result[i++] =
+                new FileStatus(0, false, 0, 0, creationTime, creationTime,
+                    PERMISSION_555, ugi.getShortUserName(),
+                    ugi.getPrimaryGroupName(), link.getTargetLink(),
+                    path);
+            continue;
+          }
+
+          //  We will represent as non-symlinks. Here it will show target
+          //  directory/file properties like permissions, isDirectory etc on
+          //  mount path. The path will be a mount link path and isDirectory is
+          //  true if target is dir, otherwise false.
+          String linkedPath = link.getTargetFileSystem().getUri().getPath();
+          if ("".equals(linkedPath)) {
+            linkedPath = "/";
+          }
           try {
-            String linkedPath = link.getTargetFileSystem().getUri().getPath();
-            if("".equals(linkedPath)) {
-              linkedPath = "/";
-            }
             FileStatus status =
                 ((ChRootedFileSystem)link.getTargetFileSystem())
                 .getMyFs().getFileStatus(new Path(linkedPath));
-            result[i++] = new FileStatus(status.getLen(), false,
-              status.getReplication(), status.getBlockSize(),
-              status.getModificationTime(), status.getAccessTime(),
-              status.getPermission(), status.getOwner(), status.getGroup(),
-              link.getTargetLink(),
-              new Path(inode.fullPath).makeQualified(
-                  myUri, null));
+            result[i++] = new FileStatus(status.getLen(), status.isDirectory(),
+                status.getReplication(), status.getBlockSize(),
+                status.getModificationTime(), status.getAccessTime(),
+                status.getPermission(), status.getOwner(), status.getGroup(),
+                null, path);
           } catch (FileNotFoundException ex) {
-            result[i++] = new FileStatus(0, false, 0, 0,
-              creationTime, creationTime, PERMISSION_555,
-              ugi.getShortUserName(), ugi.getPrimaryGroupName(),
-              link.getTargetLink(),
-              new Path(inode.fullPath).makeQualified(
-                  myUri, null));
+            LOG.warn("Cannot get one of the children's(" + path
+                + ")  target path(" + link.getTargetFileSystem().getUri()
+                + ") file status.", ex);
+            throw ex;
           }
         } else {
-          result[i++] = new FileStatus(0, true, 0, 0,
-            creationTime, creationTime, PERMISSION_555,
-            ugi.getShortUserName(), ugi.getPrimaryGroupName(),
-            new Path(inode.fullPath).makeQualified(
-                myUri, null));
+          result[i++] =
+              new FileStatus(0, true, 0, 0, creationTime, creationTime,
+                  PERMISSION_555, ugi.getShortUserName(),
+                  ugi.getPrimaryGroupName(), path);
         }
       }
       if (fallbackStatuses.length > 0) {
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
index 36f9cd1..672022b 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
@@ -59,9 +59,9 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException;
  * data to mount with other hdfs and object store clusters(hdfs://NN1,
  * o3fs://bucket1.volume1/, s3a://bucket1/)
  *
- * fs.viewfs.mounttable.Cluster./user = hdfs://NN1/user
- * fs.viewfs.mounttable.Cluster./data = o3fs://bucket1.volume1/data
- * fs.viewfs.mounttable.Cluster./backup = s3a://bucket1/backup/
+ * fs.viewfs.mounttable.Cluster.link./user = hdfs://NN1/user
+ * fs.viewfs.mounttable.Cluster.link./data = o3fs://bucket1.volume1/data
+ * fs.viewfs.mounttable.Cluster.link./backup = s3a://bucket1/backup/
  *
  * Op1: Create file hdfs://Cluster/user/fileA will go to hdfs://NN1/user/fileA
  * Op2: Create file hdfs://Cluster/data/datafile will go to
@@ -75,15 +75,19 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException;
  * data to mount with other hdfs and object store clusters
  * (hdfs://NN1, o3fs://bucket1.volume1/)
  *
- * fs.viewfs.mounttable.bucketA./user = hdfs://NN1/user
- * fs.viewfs.mounttable.bucketA./data = o3fs://bucket1.volume1/data
- * fs.viewfs.mounttable.bucketA./salesDB = s3a://bucketA/salesDB/
+ * fs.viewfs.mounttable.bucketA.link./user = hdfs://NN1/user
+ * fs.viewfs.mounttable.bucketA.link./data = o3fs://bucket1.volume1/data
+ * fs.viewfs.mounttable.bucketA.link./salesDB = s3a://bucketA/salesDB/
  *
  * Op1: Create file s3a://bucketA/user/fileA will go to hdfs://NN1/user/fileA
  * Op2: Create file s3a://bucketA/data/datafile will go to
  *      o3fs://bucket1.volume1/data/datafile
  * Op3: Create file s3a://bucketA/salesDB/dbfile will go to
  *      s3a://bucketA/salesDB/dbfile
+ *
+ * Note: In ViewFileSystemOverloadScheme, by default the mount links will be
+ * represented as non-symlinks. If you want to change this behavior, please see
+ * {@link ViewFileSystem#listStatus(Path)}
  *****************************************************************************/
 @InterfaceAudience.LimitedPrivate({ "MapReduce", "HBase", "Hive" })
 @InterfaceStability.Evolving
@@ -107,6 +111,10 @@ public class ViewFileSystemOverloadScheme extends 
ViewFileSystem {
     }
     String mountTableConfigPath =
         conf.get(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH);
+    /* The default value to false in ViewFSOverloadScheme */
+    conf.setBoolean(Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS,
+        conf.getBoolean(Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS,
+            false));
     if (null != mountTableConfigPath) {
       MountTableConfigLoader loader = new HCFSMountTableConfigLoader();
       loader.load(mountTableConfigPath, conf);
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 4578a4c..fae5d1b 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.fs.viewfs;
 
+import static 
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS;
+import static 
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT;
 import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
 
 import java.io.FileNotFoundException;
@@ -67,7 +69,8 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.Time;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * ViewFs (extends the AbstractFileSystem interface) implements a client-side
@@ -154,6 +157,7 @@ import org.apache.hadoop.util.Time;
 @InterfaceAudience.Public
 @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable 
*/
 public class ViewFs extends AbstractFileSystem {
+  static final Logger LOG = LoggerFactory.getLogger(ViewFs.class);
   final long creationTime; // of the the mount table
   final UserGroupInformation ugi; // the user/group of user who created mtable
   final Configuration config;
@@ -161,6 +165,7 @@ public class ViewFs extends AbstractFileSystem {
   Path homeDir = null;
   private ViewFileSystem.RenameStrategy renameStrategy =
       ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT;
+  private static boolean showMountLinksAsSymlinks = true;
 
   static AccessControlException readOnlyMountTable(final String operation,
       final String p) {
@@ -209,6 +214,9 @@ public class ViewFs extends AbstractFileSystem {
     creationTime = Time.now();
     ugi = UserGroupInformation.getCurrentUser();
     config = conf;
+    showMountLinksAsSymlinks = config
+        .getBoolean(CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS,
+            CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT);
     // Now build  client side view (i.e. client side mount table) from config.
     String authority = theUri.getAuthority();
     fsState = new InodeTree<AbstractFileSystem>(conf, authority) {
@@ -453,12 +461,17 @@ public class ViewFs extends AbstractFileSystem {
    * the target path FileStatus object. The target path will be available via
    * getSymlink on that children's FileStatus object. Since it represents as
    * symlink, isDirectory on that children's FileStatus will return false.
+   * This behavior can be changed by setting an advanced configuration
+   * fs.viewfs.mount.links.as.symlinks to false. In this case, mount points 
will
+   * be represented as non-symlinks and all the file/directory attributes like
+   * permissions, isDirectory etc will be assigned from it's resolved target
+   * directory/file.
    *
    * If you want to get the FileStatus of target path for that children, you 
may
    * want to use GetFileStatus API with that children's symlink path. Please 
see
    * {@link ViewFs#getFileStatus(Path f)}
    *
-   * Note: In ViewFs, the mount links are represented as symlinks.
+   * Note: In ViewFs, by default the mount links are represented as symlinks.
    */
   @Override
   public FileStatus[] listStatus(final Path f) throws AccessControlException,
@@ -999,8 +1012,7 @@ public class ViewFs extends AbstractFileSystem {
      * will be listed in the returned result.
      */
     @Override
-    public FileStatus[] listStatus(final Path f) throws AccessControlException,
-        IOException {
+    public FileStatus[] listStatus(final Path f) throws IOException {
       checkPathIsSlash(f);
       FileStatus[] fallbackStatuses = listStatusForFallbackLink();
       FileStatus[] result = new 
FileStatus[theInternalDir.getChildren().size()];
@@ -1008,37 +1020,51 @@ public class ViewFs extends AbstractFileSystem {
       for (Entry<String, INode<AbstractFileSystem>> iEntry :
           theInternalDir.getChildren().entrySet()) {
         INode<AbstractFileSystem> inode = iEntry.getValue();
-
-        
+        Path path = new Path(inode.fullPath).makeQualified(myUri, null);
         if (inode.isLink()) {
           INodeLink<AbstractFileSystem> link = 
             (INodeLink<AbstractFileSystem>) inode;
 
+          if (showMountLinksAsSymlinks) {
+            // To maintain backward compatibility, with default option(showing
+            // mount links as symlinks), we will represent target link as
+            // symlink and rest other properties are belongs to mount link 
only.
+            result[i++] =
+                new FileStatus(0, false, 0, 0, creationTime, creationTime,
+                    PERMISSION_555, ugi.getShortUserName(),
+                    ugi.getPrimaryGroupName(), link.getTargetLink(),
+                    path);
+            continue;
+          }
+
+          //  We will represent as non-symlinks. Here it will show target
+          //  directory/file properties like permissions, isDirectory etc on
+          //  mount path. The path will be a mount link path and isDirectory is
+          //  true if target is dir, otherwise false.
+          String linkedPath = link.getTargetFileSystem().getUri().getPath();
+          if ("".equals(linkedPath)) {
+            linkedPath = "/";
+          }
           try {
-            String linkedPath = link.getTargetFileSystem().getUri().getPath();
-            FileStatus status = ((ChRootedFs)link.getTargetFileSystem())
-                .getMyFs().getFileStatus(new Path(linkedPath));
-            result[i++] = new FileStatus(status.getLen(), false,
-              status.getReplication(), status.getBlockSize(),
-              status.getModificationTime(), status.getAccessTime(),
-              status.getPermission(), status.getOwner(), status.getGroup(),
-              link.getTargetLink(),
-              new Path(inode.fullPath).makeQualified(
-                  myUri, null));
+            FileStatus status =
+                ((ChRootedFs) link.getTargetFileSystem()).getMyFs()
+                    .getFileStatus(new Path(linkedPath));
+            result[i++] = new FileStatus(status.getLen(), status.isDirectory(),
+                status.getReplication(), status.getBlockSize(),
+                status.getModificationTime(), status.getAccessTime(),
+                status.getPermission(), status.getOwner(), status.getGroup(),
+                null, path);
           } catch (FileNotFoundException ex) {
-            result[i++] = new FileStatus(0, false, 0, 0,
-              creationTime, creationTime, PERMISSION_555,
-              ugi.getShortUserName(), ugi.getPrimaryGroupName(),
-              link.getTargetLink(),
-              new Path(inode.fullPath).makeQualified(
-                  myUri, null));
+            LOG.warn("Cannot get one of the children's(" + path
+                + ")  target path(" + link.getTargetFileSystem().getUri()
+                + ") file status.", ex);
+            throw ex;
           }
         } else {
-          result[i++] = new FileStatus(0, true, 0, 0,
-            creationTime, creationTime,
-            PERMISSION_555, ugi.getShortUserName(), ugi.getGroupNames()[0],
-            new Path(inode.fullPath).makeQualified(
-                myUri, null));
+          result[i++] =
+              new FileStatus(0, true, 0, 0, creationTime, creationTime,
+                  PERMISSION_555, ugi.getShortUserName(),
+                  ugi.getGroupNames()[0], path);
         }
       }
       if (fallbackStatuses.length > 0) {
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java
new file mode 100644
index 0000000..0cf6914
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeListStatus.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * ViewFsOverloadScheme ListStatus.
+ */
+public class TestViewFsOverloadSchemeListStatus {
+
+  private static final File TEST_DIR =
+      GenericTestUtils.getTestDir(TestViewfsFileStatus.class.getSimpleName());
+
+  @Before
+  public void setUp() {
+    FileUtil.fullyDelete(TEST_DIR);
+    assertTrue(TEST_DIR.mkdirs());
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    FileUtil.fullyDelete(TEST_DIR);
+  }
+
+  /**
+   * Tests the ACL and isDirectory returned from listStatus for directories and
+   * files.
+   */
+  @Test
+  public void testListStatusACL() throws IOException, URISyntaxException {
+    String testfilename = "testFileACL";
+    String childDirectoryName = "testDirectoryACL";
+    TEST_DIR.mkdirs();
+    File infile = new File(TEST_DIR, testfilename);
+    final byte[] content = "dingos".getBytes();
+
+    try (FileOutputStream fos = new FileOutputStream(infile)) {
+      fos.write(content);
+    }
+    assertEquals(content.length, infile.length());
+    File childDir = new File(TEST_DIR, childDirectoryName);
+    childDir.mkdirs();
+
+    Configuration conf = new Configuration();
+    ConfigUtil.addLink(conf, "/file", infile.toURI());
+    ConfigUtil.addLink(conf, "/dir", childDir.toURI());
+    String fileScheme = "file";
+    conf.set(String.format("fs.%s.impl", fileScheme),
+        ViewFileSystemOverloadScheme.class.getName());
+    conf.set(String
+        .format(FsConstants.FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN,
+            fileScheme), LocalFileSystem.class.getName());
+    String fileUriStr = "file:///";
+    try (FileSystem vfs = FileSystem.get(new URI(fileUriStr), conf)) {
+      assertEquals(ViewFileSystemOverloadScheme.class, vfs.getClass());
+      FileStatus[] statuses = vfs.listStatus(new Path("/"));
+
+      FileSystem localFs = ((ViewFileSystemOverloadScheme) vfs)
+          .getRawFileSystem(new Path(fileUriStr), conf);
+      FileStatus fileStat = localFs.getFileStatus(new Path(infile.getPath()));
+      FileStatus dirStat = localFs.getFileStatus(new Path(childDir.getPath()));
+
+      for (FileStatus status : statuses) {
+        if (status.getPath().getName().equals(fileScheme)) {
+          assertEquals(fileStat.getPermission(), status.getPermission());
+        } else {
+          assertEquals(dirStat.getPermission(), status.getPermission());
+        }
+      }
+
+      localFs.setPermission(new Path(infile.getPath()),
+          FsPermission.valueOf("-rwxr--r--"));
+      localFs.setPermission(new Path(childDir.getPath()),
+          FsPermission.valueOf("-r--rwxr--"));
+
+      statuses = vfs.listStatus(new Path("/"));
+      for (FileStatus status : statuses) {
+        if (status.getPath().getName().equals(fileScheme)) {
+          assertEquals(FsPermission.valueOf("-rwxr--r--"),
+              status.getPermission());
+          assertFalse(status.isDirectory());
+        } else {
+          assertEquals(FsPermission.valueOf("-r--rwxr--"),
+              status.getPermission());
+          assertTrue(status.isDirectory());
+        }
+      }
+    }
+  }
+
+  @AfterClass
+  public static void cleanup() throws IOException {
+    FileUtil.fullyDelete(TEST_DIR);
+  }
+
+}
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
index 29fcc22..7555745 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
@@ -121,7 +121,7 @@ public class TestViewfsFileStatus {
     Configuration conf = new Configuration();
     ConfigUtil.addLink(conf, "/file", infile.toURI());
     ConfigUtil.addLink(conf, "/dir", childDir.toURI());
-
+    conf.setBoolean(Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS, false);
     try (FileSystem vfs = FileSystem.get(FsConstants.VIEWFS_URI, conf)) {
       assertEquals(ViewFileSystem.class, vfs.getClass());
       FileStatus[] statuses = vfs.listStatus(new Path("/"));
@@ -148,9 +148,11 @@ public class TestViewfsFileStatus {
         if (status.getPath().getName().equals("file")) {
           assertEquals(FsPermission.valueOf("-rwxr--r--"),
               status.getPermission());
+          assertFalse(status.isDirectory());
         } else {
           assertEquals(FsPermission.valueOf("-r--rwxr--"),
               status.getPermission());
+          assertTrue(status.isDirectory());
         }
       }
     }
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFsOverloadScheme.md 
b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFsOverloadScheme.md
index 5fd8633..e65c545 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFsOverloadScheme.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFsOverloadScheme.md
@@ -28,7 +28,7 @@ View File System Overload Scheme
 
 ### Details
 
-The View File System Overload Scheme is an extension to the View File System. 
This will allow users to continue to use their existing fs.defaultFS configured 
scheme or any new scheme name instead of using scheme `viewfs`. Mount link 
configurations key, value formats are same as in [ViewFS Guide](./ViewFs.html). 
If a user wants to continue use the same fs.defaultFS and wants to have more 
mount points, then mount link configurations should have the current 
fs.defaultFS authority name as mo [...]
+The View File System Overload Scheme is an extension to the View File System. 
This will allow users to continue to use their existing fs.defaultFS configured 
scheme or any new scheme name instead of using scheme `viewfs`. Mount link 
configurations key, value formats are same as in [ViewFS Guide](./ViewFs.html). 
If a user wants to continue use the same fs.defaultFS and wants to have more 
mount points, then mount link configurations should have the current 
fs.defaultFS authority name as mo [...]
 
 Another important improvement with the ViewFileSystemOverloadScheme is, 
administrators need not copy the `mount-table.xml` configuration file to 1000s 
of client nodes. Instead they can keep the mount-table configuration file in a 
Hadoop compatible file system. So, keeping the configuration file in a central 
place makes administrators life easier as they can update mount-table in single 
place.
 
@@ -55,59 +55,59 @@ Here `<scheme>` should be same as the uri-scheme configured 
in fs.defautFS. For
 
 **Example 1:**
 
-If users want some of their existing cluster (`hdfs://mycluster`) data to 
mount with hdfs(`hdfs://mycluster`) and other object store 
clusters(`o3fs://bucket1.volume1.omhost/`, `s3a://bucket1/`), the following 
example configurations can show how to add mount links.
+If users want some of their existing cluster (`hdfs://cluster`) data to mount 
with hdfs(`hdfs://cluster`) and other object store 
clusters(`o3fs://bucket1.volume1.omhost/`, `s3a://bucket1/`), the following 
example configurations can show how to add mount links.
 
 
 ```xml
 <property>
-  <name>fs.viewfs.mounttable.Cluster./user</name>
-  <value>hdfs://mycluster/user</value>
+  <name>fs.viewfs.mounttable.cluster.link./user</name>
+  <value>hdfs://cluster/user</value>
 </property>
 
 <property>
-  <name>fs.viewfs.mounttable.Cluster./data</name>
+  <name>fs.viewfs.mounttable.cluster.link./data</name>
   <value>o3fs://bucket1.volume1/data</value>
 </property>
 
 <property>
-  <name>fs.viewfs.mounttable.Cluster./backup</name>
+  <name>fs.viewfs.mounttable.cluster.link./backup</name>
   <value>s3a://bucket1/backup/</value>
 </property>
 ```
 
 Let's consider the following operations to understand where these operations 
will be delegated based on mount links.
 
-  *Op1:* Create a file with the the path `hdfs://mycluster/user/fileA`, then 
physically this file will be created at `hdfs://mycluster/user/fileA`. This 
delegation happened based on the first configuration parameter in above 
configurations. Here `/user` mapped to `hdfs://mycluster/user/`.
+  *Op1:* Create a file with the the path `hdfs://cluster/user/fileA`, then 
physically this file will be created at `hdfs://cluster/user/fileA`. This 
delegation happened based on the first configuration parameter in above 
configurations. Here `/user` mapped to `hdfs://cluster/user/`.
 
-  *Op2:* Create a file the the path `hdfs://mycluster/data/datafile`, then 
this file will be created at `o3fs://bucket1.volume1.omhost/data/datafile`. 
This delegation happened based on second configurations parameter in above 
configurations. Here `/data` was mapped with 
`o3fs://bucket1.volume1.omhost/data/`.
+  *Op2:* Create a file the the path `hdfs://cluster/data/datafile`, then this 
file will be created at `o3fs://bucket1.volume1.omhost/data/datafile`. This 
delegation happened based on second configurations parameter in above 
configurations. Here `/data` was mapped with 
`o3fs://bucket1.volume1.omhost/data/`.
 
-  *Op3:* Create a file with the the path `hdfs://Cluster/backup/data.zip`, 
then  physically this file will be created at `s3a://bucket1/backup/data.zip`. 
This delegation happened based on the third configuration parameter in above 
configurations. Here `/backup` was mapped to `s3a://bucket1/backup/`.
+  *Op3:* Create a file with the the path `hdfs://cluster/backup/data.zip`, 
then  physically this file will be created at `s3a://bucket1/backup/data.zip`. 
This delegation happened based on the third configuration parameter in above 
configurations. Here `/backup` was mapped to `s3a://bucket1/backup/`.
 
 
 **Example 2:**
 
-If users want some of their existing cluster (`s3a://bucketA/`) data to mount 
with other hdfs cluster(`hdfs://Cluster`) and object store 
clusters(`o3fs://bucket1.volume1.omhost/`, `s3a://bucketA/`), the following 
example configurations can show how to add mount links.
+If users want some of their existing cluster (`s3a://bucketA/`) data to mount 
with other hdfs cluster(`hdfs://cluster`) and object store 
clusters(`o3fs://bucket1.volume1.omhost/`, `s3a://bucketA/`), the following 
example configurations can show how to add mount links.
 
 
 ```xml
 <property>
-  <name>fs.viewfs.mounttable.bucketA./user</name>
-  <value>hdfs://Cluster/user</value>
+  <name>fs.viewfs.mounttable.bucketA.link./user</name>
+  <value>hdfs://cluster/user</value>
 </property>
 
 <property>
-  <name>fs.viewfs.mounttable.bucketA./data</name>
+  <name>fs.viewfs.mounttable.bucketA.link./data</name>
   <value>o3fs://bucket1.volume1.omhost/data</value>
 </property>
 
 <property>
-  <name>fs.viewfs.mounttable.bucketA./salesDB</name>
+  <name>fs.viewfs.mounttable.bucketA.link./salesDB</name>
   <value>s3a://bucketA/salesDB/</value>
 </property>
 ```
 Let's consider the following operations to understand to where these 
operations will be delegated based on mount links.
 
- *Op1:* Create a file with the the path `s3a://bucketA/user/fileA`, then this 
file will be created physically at `hdfs://Cluster/user/fileA`. This delegation 
happened based on the first configuration parameter in above configurations. 
Here `/user` mapped to `hdfs://Cluster/user`.
+ *Op1:* Create a file with the the path `s3a://bucketA/user/fileA`, then this 
file will be created physically at `hdfs://cluster/user/fileA`. This delegation 
happened based on the first configuration parameter in above configurations. 
Here `/user` mapped to `hdfs://cluster/user`.
 
   *Op2:* Create a file the the path `s3a://bucketA/data/datafile`, then this 
file will be created at `o3fs://bucket1.volume1.omhost/data/datafile`. This 
delegation happened based on second configurations parameter in above 
configurations. Here `/data` was mapped with 
`o3fs://bucket1.volume1.omhost/data/`.
 
@@ -119,16 +119,18 @@ The following picture shows how the different schemes can 
be used in ViewFileSys
 
 <img src="./images/ViewFSOverloadScheme.png" width="1050" height="550"/>
 
+Note: In ViewFsOverloadScheme, by default the mount links will not be 
represented as symlinks. The permission bits and isDirectory value will be 
propagated from the target directory/file.
+
 ### Central Mount Table Configurations
 
-To enable central mount table configuration, we need to configure 
`fs.viewfs.mounttable.path` in `core-site.xml` with the value as the Hadoop 
compatible file system directory/file path, where the 
`mount-table-<versionNumber>.xml` file copied. Here versionNumber is an integer 
number and need to increase the version number and upload new file in same 
directory.
+To enable central mount table configuration, we need to configure 
`fs.viewfs.mounttable.path` in `core-site.xml` with the value as the Hadoop 
compatible file system directory/file path, where the 
`mount-table.<versionNumber>.xml` file copied. Here versionNumber is an integer 
number and need to increase the version number and upload new file in same 
directory.
 
-The ViewFileSystemOverloadScheme always loads the highest version number 
`mount-table-<versionNumber>.xml`. Please don't replace the file with same 
name. Always increment the version number to take new file picked by newly 
initializing clients. Why we don't recommend to replace the files is that, some 
client might have already opened the connections to old mount-table files 
already and in middle of loading configuration files, and replacing files can 
make them fail.
+The ViewFileSystemOverloadScheme always loads the highest version number 
`mount-table.<versionNumber>.xml`. Please don't replace the file with same 
name. Always increment the version number to take new file picked by newly 
initializing clients. Why we don't recommend to replace the files is that, some 
client might have already opened the connections to old mount-table files 
already and in middle of loading configuration files, and replacing files can 
make them fail.
 
 ```xml
 <property>
   <name>fs.viewfs.mounttable.path</name>
-  <value>hdfs://Cluster/config/mount-table-dir</value>
+  <value>hdfs://cluster/config/mount-table-dir</value>
 </property>
 ```
  If you are sure, you will never do updates to mount-table file, you can also 
configure file path directly like below. If you configure file path, it will 
not check any highest version number loading. Whatever file configured it will 
be loaded. However file name format should be same.
@@ -136,12 +138,12 @@ The ViewFileSystemOverloadScheme always loads the highest 
version number `mount-
 ```xml
 <property>
   <name>fs.viewfs.mounttable.path</name>
-  
<value>hdfs://Cluster/config/mount-table-dir/mount-table-<versionNumber>.xml</value>
+  
<value>hdfs://cluster/config/mount-table-dir/mount-table.<versionNumber>.xml</value>
 </property>
 ```
 Note: we recommend not to configure mount-links in `core-site.xml` if you 
configure above valid path. Otherwise both mount links will be mixed and can 
lead to a confused behavior.
 
-If you copy the `mount-table-<versionNumber>.xml`, you may consider having big 
replication factor depending on your cluster size. So, that file will be 
available locally to majority of clients as applications(MR/YARN/HBASE..etc) 
use locality on HDFS when reading `mount-table-<versionNumber>.xml`.
+If you copy the `mount-table.<versionNumber>.xml`, you may consider having big 
replication factor depending on your cluster size. So, that file will be 
available locally to majority of clients as applications(MR/YARN/HBASE..etc) 
use locality on HDFS when reading `mount-table.<versionNumber>.xml`.
 
 DFSAdmin commands with View File System Overload Scheme
 -------------------------------------------------------
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java
index 03c29c9..e7e74d1 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java
@@ -116,6 +116,11 @@ public class 
TestViewFileSystemOverloadSchemeHdfsFileSystemContract
     assumeTrue(rootDirTestEnabled());
     Path dir = path("/");
     Path child = path("/FileSystemContractBaseTest");
+    try (FileSystem dfs = ((ViewFileSystemOverloadScheme) fs).getRawFileSystem(
+        new Path(conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY), "/"),
+        conf)) {
+      dfs.mkdirs(child);
+    }
     assertListStatusFinds(dir, child);
   }
 
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
index 3860fa4..417a80c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
@@ -191,8 +191,17 @@ public class 
TestViewFileSystemOverloadSchemeWithHdfsScheme {
         new String[] {hdfsTargetPath.toUri().toString(),
             localTargetDir.toURI().toString() },
         conf);
+    try (DistributedFileSystem dfs = new DistributedFileSystem()) {
+      dfs.initialize(defaultFSURI, conf);
+      dfs.mkdirs(hdfsTargetPath);
+    }
 
+    try (RawLocalFileSystem lfs = new RawLocalFileSystem()) {
+      lfs.initialize(localTargetDir.toURI(), conf);
+      lfs.mkdirs(new Path(localTargetDir.toURI()));
+    }
     try (FileSystem fs = FileSystem.get(conf)) {
+      fs.mkdirs(hdfsTargetPath);
       FileStatus[] ls = fs.listStatus(new Path("/"));
       Assert.assertEquals(2, ls.length);
       String lsPath1 =


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

Reply via email to