HDFS-14096. [SPS] : Add Support for Storage Policy Satisfier in ViewFs. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/788e7473 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/788e7473 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/788e7473 Branch: refs/heads/HDFS-12943 Commit: 788e7473a404fa074b3af522416ee3d2fae865a0 Parents: 04c0347 Author: Surendra Singh Lilhore <surendralilh...@apache.org> Authored: Mon Dec 17 11:24:57 2018 +0530 Committer: Surendra Singh Lilhore <surendralilh...@apache.org> Committed: Mon Dec 17 11:24:57 2018 +0530 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/AbstractFileSystem.java | 10 ++++++++++ .../java/org/apache/hadoop/fs/FileContext.java | 18 ++++++++++++++++++ .../java/org/apache/hadoop/fs/FileSystem.java | 10 ++++++++++ .../org/apache/hadoop/fs/FilterFileSystem.java | 5 +++++ .../main/java/org/apache/hadoop/fs/FilterFs.java | 5 +++++ .../hadoop/fs/viewfs/ChRootedFileSystem.java | 5 +++++ .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 5 +++++ .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 13 +++++++++++++ .../java/org/apache/hadoop/fs/viewfs/ViewFs.java | 12 ++++++++++++ .../org/apache/hadoop/fs/TestHarFileSystem.java | 2 ++ .../hadoop/fs/viewfs/ViewFileSystemBaseTest.java | 5 +++++ .../src/main/java/org/apache/hadoop/fs/Hdfs.java | 5 +++++ .../apache/hadoop/hdfs/DistributedFileSystem.java | 6 +----- 13 files changed, 96 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index 721f009..d1e50d0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -1255,6 +1255,16 @@ public abstract class AbstractFileSystem { } /** + * Set the source path to satisfy storage policy. + * @param path The source path referring to either a directory or a file. + * @throws IOException + */ + public void satisfyStoragePolicy(final Path path) throws IOException { + throw new UnsupportedOperationException( + getClass().getSimpleName() + " doesn't support satisfyStoragePolicy"); + } + + /** * Set the storage policy for a given file or directory. * * @param path file or directory path. http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 084f8f1..e5438f5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -2781,6 +2781,24 @@ public class FileContext { } /** + * Set the source path to satisfy storage policy. + * @param path The source path referring to either a directory or a file. + * @throws IOException + */ + public void satisfyStoragePolicy(final Path path) + throws IOException { + final Path absF = fixRelativePart(path); + new FSLinkResolver<Void>() { + @Override + public Void next(final AbstractFileSystem fs, final Path p) + throws IOException { + fs.satisfyStoragePolicy(path); + return null; + } + }.resolve(this, absF); + } + + /** * Set the storage policy for a given file or directory. * * @param path file or directory path. http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index fe4159b..5454cd0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -3086,6 +3086,16 @@ public abstract class FileSystem extends Configured } /** + * Set the source path to satisfy storage policy. + * @param path The source path referring to either a directory or a file. + * @throws IOException + */ + public void satisfyStoragePolicy(final Path path) throws IOException { + throw new UnsupportedOperationException( + getClass().getSimpleName() + " doesn't support setStoragePolicy"); + } + + /** * Set the storage policy for a given file or directory. * * @param src file or directory path. http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index f9bbfb1..a3e10b4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -646,6 +646,11 @@ public class FilterFileSystem extends FileSystem { } @Override + public void satisfyStoragePolicy(Path src) throws IOException { + fs.satisfyStoragePolicy(src); + } + + @Override public void setStoragePolicy(Path src, String policyName) throws IOException { fs.setStoragePolicy(src, policyName); http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index b2a9aab..d6dc59b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -406,6 +406,11 @@ public abstract class FilterFs extends AbstractFileSystem { } @Override + public void satisfyStoragePolicy(final Path path) throws IOException { + myFs.satisfyStoragePolicy(path); + } + + @Override public void setStoragePolicy(Path path, String policyName) throws IOException { myFs.setStoragePolicy(path, policyName); http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java index 8b90f53..b195be3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java @@ -450,6 +450,11 @@ class ChRootedFileSystem extends FilterFileSystem { } @Override + public void satisfyStoragePolicy(Path src) throws IOException { + super.satisfyStoragePolicy(fullPath(src)); + } + + @Override public void setStoragePolicy(Path src, String policyName) throws IOException { super.setStoragePolicy(fullPath(src), policyName); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index 6168df0..5e14be1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -399,6 +399,11 @@ class ChRootedFs extends AbstractFileSystem { } @Override + public void satisfyStoragePolicy(final Path path) throws IOException { + myFs.satisfyStoragePolicy(path); + } + + @Override public void setStoragePolicy(Path path, String policyName) throws IOException { myFs.setStoragePolicy(fullPath(path), policyName); http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 9523070..bc73f1b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -811,6 +811,13 @@ public class ViewFileSystem extends FileSystem { } @Override + public void satisfyStoragePolicy(Path src) throws IOException { + InodeTree.ResolveResult<FileSystem> res = + fsState.resolve(getUriPath(src), true); + res.targetFileSystem.satisfyStoragePolicy(res.remainingPath); + } + + @Override public void setStoragePolicy(Path src, String policyName) throws IOException { InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src), true); @@ -1246,6 +1253,12 @@ public class ViewFileSystem extends FileSystem { } @Override + public void satisfyStoragePolicy(Path src) throws IOException { + checkPathIsSlash(src); + throw readOnlyMountTable("satisfyStoragePolicy", src); + } + + @Override public void setStoragePolicy(Path src, String policyName) throws IOException { checkPathIsSlash(src); http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 8366eac..1770476 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -752,6 +752,13 @@ public class ViewFs extends AbstractFileSystem { } @Override + public void satisfyStoragePolicy(final Path path) throws IOException { + InodeTree.ResolveResult<AbstractFileSystem> res = + fsState.resolve(getUriPath(path), true); + res.targetFileSystem.satisfyStoragePolicy(res.remainingPath); + } + + @Override public void setStoragePolicy(final Path path, final String policyName) throws IOException { InodeTree.ResolveResult<AbstractFileSystem> res = @@ -1155,6 +1162,11 @@ public class ViewFs extends AbstractFileSystem { } @Override + public void satisfyStoragePolicy(final Path path) throws IOException { + throw readOnlyMountTable("satisfyStoragePolicy", path); + } + + @Override public void setStoragePolicy(Path path, String policyName) throws IOException { throw readOnlyMountTable("setStoragePolicy", path); http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java index 025b831..b3021a9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java @@ -213,6 +213,8 @@ public class TestHarFileSystem { public void access(Path path, FsAction mode) throws IOException; + void satisfyStoragePolicy(Path src) throws IOException; + public void setStoragePolicy(Path src, String policyName) throws IOException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index b6158dfc..b9453df 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -968,6 +968,11 @@ abstract public class ViewFileSystemBaseTest { fsView.unsetStoragePolicy(new Path("/internalDir")); } + @Test(expected = AccessControlException.class) + public void testInternalSatisfyStoragePolicy() throws IOException { + fsView.satisfyStoragePolicy(new Path("/internalDir")); + } + @Test(expected = NotInMountpointException.class) public void testInternalgetStoragePolicy() throws IOException { fsView.getStoragePolicy(new Path("/internalDir")); http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java index d07d5a2..290f2c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -490,6 +490,11 @@ public class Hdfs extends AbstractFileSystem { } @Override + public void satisfyStoragePolicy(Path path) throws IOException { + dfs.satisfyStoragePolicy(getUriPath(path)); + } + + @Override public void setStoragePolicy(Path path, String policyName) throws IOException { dfs.setStoragePolicy(getUriPath(path), policyName); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 7dd02bd..2b0b2c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2870,11 +2870,7 @@ public class DistributedFileSystem extends FileSystem } /** - * Set the source path to satisfy storage policy. This API is non-recursive - * in nature, i.e., if the source path is a directory then all the files - * immediately under the directory would be considered for satisfying the - * policy and the sub-directories if any under this path will be skipped. - * + * Set the source path to satisfy storage policy. * @param path The source path referring to either a directory or a file. * @throws IOException */ --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
