Repository: hadoop Updated Branches: refs/heads/branch-2 7e643130f -> 8bfb9971c refs/heads/branch-2.8 a1682876c -> 90f3108b8
HADOOP-14455. ViewFileSystem#rename should support be supported within same nameservice with different mountpoints. Contributed by Brahma Reddy Battula. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bfb9971 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bfb9971 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bfb9971 Branch: refs/heads/branch-2 Commit: 8bfb9971cad1a83da852de4d12f1de4197d25d21 Parents: 7e64313 Author: Brahma Reddy Battula <[email protected]> Authored: Sun Jul 30 14:01:41 2017 +0800 Committer: Brahma Reddy Battula <[email protected]> Committed: Sun Jul 30 14:01:41 2017 +0800 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/viewfs/Constants.java | 2 + .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 79 +++++++++++----- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 43 ++++----- .../src/main/resources/core-default.xml | 9 ++ .../conf/TestCommonConfigurationFields.java | 1 + .../hadoop/fs/contract/ContractTestUtils.java | 54 +++++++++++ .../fs/viewfs/ViewFileSystemBaseTest.java | 79 +++++++++++++--- .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 94 ++++++++++++++++---- .../fs/viewfs/TestViewFileSystemHdfs.java | 28 ++++-- 9 files changed, 310 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java index ec8ab2b..9882a8e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java @@ -66,4 +66,6 @@ public interface Constants { static public final FsPermission PERMISSION_555 = new FsPermission((short) 0555); + + String CONFIG_VIEWFS_RENAME_STRATEGY = "fs.viewfs.rename.strategy"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 6b48e72..27d8c0b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -108,7 +108,8 @@ public class ViewFileSystem extends FileSystem { Configuration config; InodeTree<FileSystem> fsState; // the fs state; ie the mount table Path homeDir = null; - + // Default to rename within same mountpoint + private RenameStrategy renameStrategy = RenameStrategy.SAME_MOUNTPOINT; /** * Make the path Absolute and get the path-part of a pathname. * Checks that URI matches this file system @@ -190,6 +191,9 @@ public class ViewFileSystem extends FileSystem { } }; workingDir = this.getHomeDirectory(); + renameStrategy = RenameStrategy.valueOf( + conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + RenameStrategy.SAME_MOUNTPOINT.toString())); } catch (URISyntaxException e) { throw new IOException("URISyntax exception: " + theUri); } @@ -477,27 +481,55 @@ public class ViewFileSystem extends FileSystem { if (resDst.isInternalDir()) { throw readOnlyMountTable("rename", dst); } - /** - // Alternate 1: renames within same file system - valid but we disallow - // Alternate 2: (as described in next para - valid but we have disallowed it - // - // Note we compare the URIs. the URIs include the link targets. - // hence we allow renames across mount links as long as the mount links - // point to the same target. - if (!resSrc.targetFileSystem.getUri().equals( - resDst.targetFileSystem.getUri())) { - throw new IOException("Renames across Mount points not supported"); - } - */ - - // - // Alternate 3 : renames ONLY within the the same mount links. - // - if (resSrc.targetFileSystem !=resDst.targetFileSystem) { - throw new IOException("Renames across Mount points not supported"); + + URI srcUri = resSrc.targetFileSystem.getUri(); + URI dstUri = resDst.targetFileSystem.getUri(); + + verifyRenameStrategy(srcUri, dstUri, + resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy); + + ChRootedFileSystem srcFS = (ChRootedFileSystem) resSrc.targetFileSystem; + ChRootedFileSystem dstFS = (ChRootedFileSystem) resDst.targetFileSystem; + return srcFS.getMyFs().rename(srcFS.fullPath(resSrc.remainingPath), + dstFS.fullPath(resDst.remainingPath)); + } + + static void verifyRenameStrategy(URI srcUri, URI dstUri, + boolean isSrcDestSame, ViewFileSystem.RenameStrategy renameStrategy) + throws IOException { + switch (renameStrategy) { + case SAME_FILESYSTEM_ACROSS_MOUNTPOINT: + if (srcUri.getAuthority() != null) { + if (!(srcUri.getScheme().equals(dstUri.getScheme()) && srcUri + .getAuthority().equals(dstUri.getAuthority()))) { + throw new IOException("Renames across Mount points not supported"); + } + } + + break; + case SAME_TARGET_URI_ACROSS_MOUNTPOINT: + // Alternate 2: Rename across mountpoints with same target. + // i.e. Rename across alias mountpoints. + // + // Note we compare the URIs. the URIs include the link targets. + // hence we allow renames across mount links as long as the mount links + // point to the same target. + if (!srcUri.equals(dstUri)) { + throw new IOException("Renames across Mount points not supported"); + } + + break; + case SAME_MOUNTPOINT: + // + // Alternate 3 : renames ONLY within the the same mount links. + // + if (!isSrcDestSame) { + throw new IOException("Renames across Mount points not supported"); + } + break; + default: + throw new IllegalArgumentException ("Unexpected rename strategy"); } - return resSrc.targetFileSystem.rename(resSrc.remainingPath, - resDst.remainingPath); } @Override @@ -1080,4 +1112,9 @@ public class ViewFileSystem extends FileSystem { throw new NotInMountpointException(f, "getQuotaUsage"); } } + + enum RenameStrategy { + SAME_MOUNTPOINT, SAME_TARGET_URI_ACROSS_MOUNTPOINT, + SAME_FILESYSTEM_ACROSS_MOUNTPOINT + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 3a34a91..364485f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -157,7 +157,9 @@ public class ViewFs extends AbstractFileSystem { final Configuration config; InodeTree<AbstractFileSystem> fsState; // the fs state; ie the mount table Path homeDir = null; - + private ViewFileSystem.RenameStrategy renameStrategy = + ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT; + static AccessControlException readOnlyMountTable(final String operation, final String p) { return new AccessControlException( @@ -237,6 +239,9 @@ public class ViewFs extends AbstractFileSystem { // return MergeFs.createMergeFs(mergeFsURIList, config); } }; + renameStrategy = ViewFileSystem.RenameStrategy.valueOf( + conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT.toString())); } @Override @@ -495,37 +500,23 @@ public class ViewFs extends AbstractFileSystem { + " is readOnly"); } - InodeTree.ResolveResult<AbstractFileSystem> resDst = + InodeTree.ResolveResult<AbstractFileSystem> resDst = fsState.resolve(getUriPath(dst), false); if (resDst.isInternalDir()) { throw new AccessControlException( "Cannot Rename within internal dirs of mount table: dest=" + dst + " is readOnly"); } - - /** - // Alternate 1: renames within same file system - valid but we disallow - // Alternate 2: (as described in next para - valid but we have disallowed it - // - // Note we compare the URIs. the URIs include the link targets. - // hence we allow renames across mount links as long as the mount links - // point to the same target. - if (!resSrc.targetFileSystem.getUri().equals( - resDst.targetFileSystem.getUri())) { - throw new IOException("Renames across Mount points not supported"); - } - */ - - // - // Alternate 3 : renames ONLY within the the same mount links. - // - - if (resSrc.targetFileSystem !=resDst.targetFileSystem) { - throw new IOException("Renames across Mount points not supported"); - } - - resSrc.targetFileSystem.renameInternal(resSrc.remainingPath, - resDst.remainingPath, overwrite); + //Alternate 1: renames within same file system + URI srcUri = resSrc.targetFileSystem.getUri(); + URI dstUri = resDst.targetFileSystem.getUri(); + ViewFileSystem.verifyRenameStrategy(srcUri, dstUri, + resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy); + + ChRootedFs srcFS = (ChRootedFs) resSrc.targetFileSystem; + ChRootedFs dstFS = (ChRootedFs) resDst.targetFileSystem; + srcFS.getMyFs().renameInternal(srcFS.fullPath(resSrc.remainingPath), + dstFS.fullPath(resDst.remainingPath), overwrite); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index f72afc5..104ba60 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -783,6 +783,15 @@ </property> <property> + <name>fs.viewfs.rename.strategy</name> + <value>SAME_MOUNTPOINT</value> + <description>Allowed rename strategy to rename between multiple mountpoints. + Allowed values are SAME_MOUNTPOINT,SAME_TARGET_URI_ACROSS_MOUNTPOINT and + SAME_FILESYSTEM_ACROSS_MOUNTPOINT. + </description> +</property> + +<property> <name>fs.AbstractFileSystem.ftp.impl</name> <value>org.apache.hadoop.fs.ftp.FtpFs</value> <description>The FileSystem for Ftp: uris.</description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index 97c6279..228ee89 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -94,6 +94,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { xmlPropsToSkipCompare.add("nfs3.server.port"); xmlPropsToSkipCompare.add("test.fs.s3.name"); xmlPropsToSkipCompare.add("test.fs.s3n.name"); + xmlPropsToSkipCompare.add("fs.viewfs.rename.strategy"); // S3/S3A properties are in a different subtree. // - org.apache.hadoop.fs.s3.S3FileSystemConfigKeys http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java index ebb8220..e4fd97c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.contract; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; @@ -691,6 +692,21 @@ public class ContractTestUtils extends Assert { /** * Assert that a file exists and whose {@link FileStatus} entry * declares that this is a file and not a symlink or directory. + * + * @param fileContext filesystem to resolve path against + * @param filename name of the file + * @throws IOException IO problems during file operations + */ + public static void assertIsFile(FileContext fileContext, Path filename) + throws IOException { + assertPathExists(fileContext, "Expected file", filename); + FileStatus status = fileContext.getFileStatus(filename); + assertIsFile(filename, status); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. * @param filename name of the file * @param status file status */ @@ -739,6 +755,25 @@ public class ContractTestUtils extends Assert { } /** + * Assert that a path exists -but make no assertions as to the + * type of that entry. + * + * @param fileContext fileContext to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws FileNotFoundException raised if the path is missing + * @throws IOException IO problems + */ + public static void assertPathExists(FileContext fileContext, String message, + Path path) throws IOException { + if (!fileContext.util().exists(path)) { + //failure, report it + throw new FileNotFoundException( + message + ": not found " + path + " in " + path.getParent()); + } + } + + /** * Assert that a path does not exist. * * @param fileSystem filesystem to examine @@ -759,6 +794,25 @@ public class ContractTestUtils extends Assert { } /** + * Assert that a path does not exist. + * + * @param fileContext fileContext to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws IOException IO problems + */ + public static void assertPathDoesNotExist(FileContext fileContext, + String message, Path path) throws IOException { + try { + FileStatus status = fileContext.getFileStatus(path); + fail(message + ": unexpectedly found " + path + " as " + status); + } catch (FileNotFoundException expected) { + //this is expected + + } + } + + /** * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry. * @param fs filesystem * @param dir directory to scan http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 8c4ddc3..5ebcacb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -27,6 +27,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import static org.apache.hadoop.fs.FileSystemTestHelper.*; @@ -51,6 +52,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -351,28 +353,83 @@ abstract public class ViewFileSystemBaseTest { } // rename across mount points that point to same target also fail - @Test(expected=IOException.class) + @Test public void testRenameAcrossMounts1() throws IOException { fileSystemTestHelper.createFile(fsView, "/user/foo"); - fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); - /* - code if we had wanted this to suceed - Assert.assertFalse(fSys.exists(new Path("/user/foo"))); - Assert.assertFalse(fSysLocal.exists(new Path(targetTestRoot,"user/foo"))); - Assert.assertTrue(fSys.isFile(FileSystemTestHelper.getTestRootPath(fSys,"/user2/fooBarBar"))); - Assert.assertTrue(fSysLocal.isFile(new Path(targetTestRoot,"user/fooBarBar"))); - */ + try { + fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); + ContractTestUtils.fail("IOException is not thrown on rename operation"); + } catch (IOException e) { + GenericTestUtils + .assertExceptionContains("Renames across Mount points not supported", + e); + } } // rename across mount points fail if the mount link targets are different // even if the targets are part of the same target FS - @Test(expected=IOException.class) + @Test public void testRenameAcrossMounts2() throws IOException { fileSystemTestHelper.createFile(fsView, "/user/foo"); - fsView.rename(new Path("/user/foo"), new Path("/data/fooBar")); + try { + fsView.rename(new Path("/user/foo"), new Path("/data/fooBar")); + ContractTestUtils.fail("IOException is not thrown on rename operation"); + } catch (IOException e) { + GenericTestUtils + .assertExceptionContains("Renames across Mount points not supported", + e); + } } - + + // RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled + // to rename across mount points that point to same target URI + @Test + public void testRenameAcrossMounts3() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT + .toString()); + FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2); + fileSystemTestHelper.createFile(fsView2, "/user/foo"); + fsView2.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); + ContractTestUtils + .assertPathDoesNotExist(fsView2, "src should not exist after rename", + new Path("/user/foo")); + ContractTestUtils + .assertPathDoesNotExist(fsTarget, "src should not exist after rename", + new Path(targetTestRoot, "user/foo")); + ContractTestUtils.assertIsFile(fsView2, + fileSystemTestHelper.getTestRootPath(fsView2, "/user2/fooBarBar")); + ContractTestUtils + .assertIsFile(fsTarget, new Path(targetTestRoot, "user/fooBarBar")); + } + + // RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled + // to rename across mount points where the mount link targets are different + // but are part of the same target FS + @Test + public void testRenameAcrossMounts4() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT + .toString()); + FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2); + fileSystemTestHelper.createFile(fsView2, "/user/foo"); + fsView2.rename(new Path("/user/foo"), new Path("/data/fooBar")); + ContractTestUtils + .assertPathDoesNotExist(fsView2, "src should not exist after rename", + new Path("/user/foo")); + ContractTestUtils + .assertPathDoesNotExist(fsTarget, "src should not exist after rename", + new Path(targetTestRoot, "user/foo")); + ContractTestUtils.assertIsFile(fsView2, + fileSystemTestHelper.getTestRootPath(fsView2, "/data/fooBar")); + ContractTestUtils + .assertIsFile(fsTarget, new Path(targetTestRoot, "data/fooBar")); + } + static protected boolean SupportsBlocks = false; // local fs use 1 block // override for HDFS @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index 50237d1..73e43e1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -58,6 +58,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.local.LocalConfigKeys; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; @@ -66,6 +67,7 @@ import org.apache.hadoop.fs.viewfs.ViewFs.MountPoint; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -345,33 +347,93 @@ abstract public class ViewFsBaseTest { } // rename across mount points that point to same target also fail - @Test(expected=IOException.class) + @Test public void testRenameAcrossMounts1() throws IOException { fileContextTestHelper.createFile(fcView, "/user/foo"); - fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); - /* - code if we had wanted this to succeed - Assert.assertFalse(exists(fc, new Path("/user/foo"))); - Assert.assertFalse(exists(fclocal, new Path(targetTestRoot,"user/foo"))); - Assert.assertTrue(isFile(fc, - FileContextTestHelper.getTestRootPath(fc,"/user2/fooBarBar"))); - Assert.assertTrue(isFile(fclocal, - new Path(targetTestRoot,"user/fooBarBar"))); - */ + try { + fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); + ContractTestUtils.fail("IOException is not thrown on rename operation"); + } catch (IOException e) { + GenericTestUtils + .assertExceptionContains("Renames across Mount points not supported", + e); + } } // rename across mount points fail if the mount link targets are different // even if the targets are part of the same target FS - @Test(expected=IOException.class) + @Test public void testRenameAcrossMounts2() throws IOException { fileContextTestHelper.createFile(fcView, "/user/foo"); - fcView.rename(new Path("/user/foo"), new Path("/data/fooBar")); + try { + fcView.rename(new Path("/user/foo"), new Path("/data/fooBar")); + ContractTestUtils.fail("IOException is not thrown on rename operation"); + } catch (IOException e) { + GenericTestUtils + .assertExceptionContains("Renames across Mount points not supported", + e); + } } - - - - + + // RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled + // to rename across mount points that point to same target URI + @Test + public void testRenameAcrossMounts3() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT + .toString()); + + FileContext fcView2 = + FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2); + String user1Path = "/user/foo"; + fileContextTestHelper.createFile(fcView2, user1Path); + String user2Path = "/user2/fooBarBar"; + Path user2Dst = new Path(user2Path); + fcView2.rename(new Path(user1Path), user2Dst); + ContractTestUtils + .assertPathDoesNotExist(fcView2, "src should not exist after rename", + new Path(user1Path)); + ContractTestUtils + .assertPathDoesNotExist(fcTarget, "src should not exist after rename", + new Path(targetTestRoot, "user/foo")); + ContractTestUtils.assertIsFile(fcView2, + fileContextTestHelper.getTestRootPath(fcView2, user2Path)); + ContractTestUtils + .assertIsFile(fcTarget, new Path(targetTestRoot, "user/fooBarBar")); + } + + // RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled + // to rename across mount points if the mount link targets are different + // but are part of the same target FS + @Test + public void testRenameAcrossMounts4() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY, + ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT + .toString()); + FileContext fcView2 = + FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2); + String userPath = "/user/foo"; + fileContextTestHelper.createFile(fcView2, userPath); + String anotherMountPath = "/data/fooBar"; + Path anotherDst = new Path(anotherMountPath); + fcView2.rename(new Path(userPath), anotherDst); + + ContractTestUtils + .assertPathDoesNotExist(fcView2, "src should not exist after rename", + new Path(userPath)); + ContractTestUtils + .assertPathDoesNotExist(fcTarget, "src should not exist after rename", + new Path(targetTestRoot, "user/foo")); + ContractTestUtils.assertIsFile(fcView2, + fileContextTestHelper.getTestRootPath(fcView2, anotherMountPath)); + ContractTestUtils + .assertIsFile(fcView2, new Path(targetTestRoot, "data/fooBar")); + } + static protected boolean SupportsBlocks = false; // local fs use 1 block // override for HDFS @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bfb9971/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java index 9221317..7a31a37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java @@ -29,15 +29,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.security.UserGroupInformation; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.*; public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { @@ -137,4 +135,24 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { int getExpectedDelegationTokenCountWithCredentials() { return 2; } + + //Rename should fail on across different fileSystems + @Test + public void testRenameAccorssFilesystem() throws IOException { + //data is mountpoint in nn1 + Path mountDataRootPath = new Path("/data"); + //mountOnNn2 is nn2 mountpoint + Path fsTargetFilePath = new Path("/mountOnNn2"); + Path filePath = new Path(mountDataRootPath + "/ttest"); + Path hdfFilepath = new Path(fsTargetFilePath + "/ttest2"); + fsView.create(filePath); + try { + fsView.rename(filePath, hdfFilepath); + ContractTestUtils.fail("Should thrown IOE on Renames across filesytems"); + } catch (IOException e) { + GenericTestUtils + .assertExceptionContains("Renames across Mount points not supported", + e); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
