HADOOP-12161. Add getStoragePolicy API to the FileSystem interface. (Contributed by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adfa34ff Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adfa34ff Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adfa34ff Branch: refs/heads/HDFS-7240 Commit: adfa34ff9992295a6d2496b259d8c483ed90b566 Parents: 3bba180 Author: Arpit Agarwal <a...@apache.org> Authored: Thu Jul 23 10:13:04 2015 -0700 Committer: Arpit Agarwal <a...@apache.org> Committed: Thu Jul 23 10:13:04 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../apache/hadoop/fs/AbstractFileSystem.java | 13 +++++++++++++ .../java/org/apache/hadoop/fs/FileContext.java | 20 ++++++++++++++++++++ .../java/org/apache/hadoop/fs/FileSystem.java | 13 +++++++++++++ .../org/apache/hadoop/fs/FilterFileSystem.java | 6 ++++++ .../java/org/apache/hadoop/fs/FilterFs.java | 6 ++++++ .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 ++++++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 15 +++++++++++++++ .../org/apache/hadoop/fs/TestHarFileSystem.java | 3 +++ .../main/java/org/apache/hadoop/fs/Hdfs.java | 5 +++++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 18 ++++++++++++++++++ .../hadoop/hdfs/DistributedFileSystem.java | 19 +++++++++++++++++++ .../hadoop/hdfs/TestBlockStoragePolicy.java | 17 +++++++++++++++++ 13 files changed, 144 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ff7d2ad..f1a3bc9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -716,6 +716,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin Walsh via Colin P. McCabe) + HADOOP-12161. Add getStoragePolicy API to the FileSystem interface. + (Brahma Reddy Battula via Arpit Agarwal) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index cb3fb86..2bc3859 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -1237,6 +1237,19 @@ public abstract class AbstractFileSystem { } /** + * Retrieve the storage policy for a given file or directory. + * + * @param src file or directory path. + * @return storage policy for give file. + * @throws IOException + */ + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + throw new UnsupportedOperationException(getClass().getSimpleName() + + " doesn't support getStoragePolicy"); + } + + /** * Retrieve all the storage policies supported by this file system. * * @return all storage policies supported by this filesystem. http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 0f21a61..a98d662 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -49,6 +49,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT; + import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RpcClientException; import org.apache.hadoop.ipc.RpcServerException; @@ -2692,6 +2693,25 @@ public class FileContext { } /** + * Query the effective storage policy ID for the given file or directory. + * + * @param src file or directory path. + * @return storage policy for give file. + * @throws IOException + */ + public BlockStoragePolicySpi getStoragePolicy(Path path) throws IOException { + final Path absF = fixRelativePart(path); + return new FSLinkResolver<BlockStoragePolicySpi>() { + @Override + public BlockStoragePolicySpi next(final AbstractFileSystem fs, + final Path p) + throws IOException { + return fs.getStoragePolicy(p); + } + }.resolve(this, absF); + } + + /** * Retrieve all the storage policies supported by this file system. * * @return all storage policies supported by this filesystem. http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 5e03e88..a01d3ea 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2629,6 +2629,19 @@ public abstract class FileSystem extends Configured implements Closeable { } /** + * Query the effective storage policy ID for the given file or directory. + * + * @param src file or directory path. + * @return storage policy for give file. + * @throws IOException + */ + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + throw new UnsupportedOperationException(getClass().getSimpleName() + + " doesn't support getStoragePolicy"); + } + + /** * Retrieve all the storage policies supported by this file system. * * @return all storage policies supported by this filesystem. http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 11f3b23..815ef69 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -628,6 +628,12 @@ public class FilterFileSystem extends FileSystem { } @Override + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + return fs.getStoragePolicy(src); + } + + @Override public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies() throws IOException { return fs.getAllStoragePolicies(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index 539b26e..248377c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -406,6 +406,12 @@ public abstract class FilterFs extends AbstractFileSystem { } @Override + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + return myFs.getStoragePolicy(src); + } + + @Override public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies() throws IOException { return myFs.getAllStoragePolicies(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index 4e5a0d5..568b9a0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -386,6 +386,12 @@ class ChRootedFs extends AbstractFileSystem { } @Override + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + return myFs.getStoragePolicy(src); + } + + @Override public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies() throws IOException { return myFs.getAllStoragePolicies(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index bec292c..6f05e77 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -34,6 +34,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.BlockStoragePolicySpi; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -748,6 +749,20 @@ public class ViewFs extends AbstractFileSystem { res.targetFileSystem.setStoragePolicy(res.remainingPath, policyName); } + /** + * Retrieve the storage policy for a given file or directory. + * + * @param src file or directory path. + * @return storage policy for give file. + * @throws IOException + */ + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + InodeTree.ResolveResult<AbstractFileSystem> res = + fsState.resolve(getUriPath(src), true); + return res.targetFileSystem.getStoragePolicy(res.remainingPath); + } + /* * An instance of this class represents an internal dir of the viewFs * ie internal dir of the mount table. http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java index 46f24fc..1710ba4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java @@ -210,6 +210,9 @@ public class TestHarFileSystem { public void setStoragePolicy(Path src, String policyName) throws IOException; + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException; + public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies() throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java index 3f78b31..ba5687c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -473,6 +473,11 @@ public class Hdfs extends AbstractFileSystem { } @Override + public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException { + return dfs.getStoragePolicy(getUriPath(src)); + } + + @Override public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies() throws IOException { return Arrays.asList(dfs.getStoragePolicies()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 6f9e613..44713a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1574,6 +1574,24 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } /** + * @return Get the storage policy for specified path + */ + public BlockStoragePolicy getStoragePolicy(String path) throws IOException { + HdfsFileStatus status = getFileInfo(path); + if (status == null) { + throw new FileNotFoundException("File does not exist: " + path); + } + byte storagePolicyId = status.getStoragePolicy(); + BlockStoragePolicy[] policies = getStoragePolicies(); + for (BlockStoragePolicy policy : policies) { + if (policy.getId() == storagePolicyId) { + return policy; + } + } + return null; + } + + /** * @return All the existing storage policies */ public BlockStoragePolicy[] getStoragePolicies() throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 902636c..4d5c0f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -557,6 +557,25 @@ public class DistributedFileSystem extends FileSystem { } @Override + public BlockStoragePolicySpi getStoragePolicy(Path path) throws IOException { + statistics.incrementReadOps(1); + Path absF = fixRelativePart(path); + + return new FileSystemLinkResolver<BlockStoragePolicySpi>() { + @Override + public BlockStoragePolicySpi doCall(final Path p) throws IOException { + return getClient().getStoragePolicy(getPathName(p)); + } + + @Override + public BlockStoragePolicySpi next(final FileSystem fs, final Path p) + throws IOException, UnresolvedLinkException { + return fs.getStoragePolicy(p); + } + }.resolve(this, absF); + } + + @Override public Collection<BlockStoragePolicy> getAllStoragePolicies() throws IOException { return Arrays.asList(dfs.getStoragePolicies()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java index 0d59ded..afd2597 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java @@ -871,9 +871,25 @@ public class TestBlockStoragePolicy { GenericTestUtils.assertExceptionContains(invalidPath.toString(), e); } + try { + fs.getStoragePolicy(invalidPath); + Assert.fail("Should throw a FileNotFoundException"); + } catch (FileNotFoundException e) { + GenericTestUtils.assertExceptionContains(invalidPath.toString(), e); + } + fs.setStoragePolicy(fooFile, HdfsServerConstants.COLD_STORAGE_POLICY_NAME); fs.setStoragePolicy(barDir, HdfsServerConstants.WARM_STORAGE_POLICY_NAME); fs.setStoragePolicy(barFile2, HdfsServerConstants.HOT_STORAGE_POLICY_NAME); + Assert.assertEquals("File storage policy should be COLD", + HdfsServerConstants.COLD_STORAGE_POLICY_NAME, + fs.getStoragePolicy(fooFile).getName()); + Assert.assertEquals("File storage policy should be WARM", + HdfsServerConstants.WARM_STORAGE_POLICY_NAME, + fs.getStoragePolicy(barDir).getName()); + Assert.assertEquals("File storage policy should be HOT", + HdfsServerConstants.HOT_STORAGE_POLICY_NAME, + fs.getStoragePolicy(barFile2).getName()); dirList = fs.getClient().listPaths(dir.toString(), HdfsFileStatus.EMPTY_NAME).getPartialListing(); @@ -1306,4 +1322,5 @@ public class TestBlockStoragePolicy { Assert.assertEquals(StorageType.ARCHIVE, i.next().getKey()); } } + }