Repository: hadoop Updated Branches: refs/heads/branch-2 dc76c4b03 -> f8932d478 refs/heads/trunk df9e7280d -> cc71ad80e
HDFS-8815. DFS getStoragePolicy implementation using single RPC call (Contributed by Surendra Singh Lilhore) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc71ad80 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc71ad80 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc71ad80 Branch: refs/heads/trunk Commit: cc71ad80e184fc6e5043729e8cfcf6a62ca3e71f Parents: df9e728 Author: Vinayakumar B <[email protected]> Authored: Thu Aug 6 11:10:48 2015 +0530 Committer: Vinayakumar B <[email protected]> Committed: Thu Aug 6 11:10:48 2015 +0530 ---------------------------------------------------------------------- .../hadoop/hdfs/protocol/ClientProtocol.java | 14 ++++++++++++ .../src/main/proto/ClientNamenodeProtocol.proto | 10 +++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 23 ++++++++++---------- ...tNamenodeProtocolServerSideTranslatorPB.java | 17 +++++++++++++++ .../ClientNamenodeProtocolTranslatorPB.java | 13 +++++++++++ .../hdfs/server/namenode/FSDirAttrOp.java | 23 ++++++++++++++++++++ .../hdfs/server/namenode/FSNamesystem.java | 19 ++++++++++++++++ .../hdfs/server/namenode/NameNodeRpcServer.java | 6 +++++ .../hadoop/hdfs/TestBlockStoragePolicy.java | 23 ++++++++++++++++++++ 10 files changed, 140 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc71ad80/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 713c23c..8528999 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -280,6 +280,20 @@ public interface ClientProtocol { throws IOException; /** + * Get the storage policy for a file/directory. + * @param path + * Path of an existing file/directory. + * @throws AccessControlException + * If access is denied + * @throws org.apache.hadoop.fs.UnresolvedLinkException + * if <code>src</code> contains a symlink + * @throws java.io.FileNotFoundException + * If file/dir <code>src</code> is not found + */ + @Idempotent + BlockStoragePolicy getStoragePolicy(String path) throws IOException; + + /** * Set permissions for an existing file/directory. * * @throws org.apache.hadoop.security.AccessControlException If access is http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc71ad80/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index b44c556..7d32568 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -112,6 +112,14 @@ message SetStoragePolicyRequestProto { message SetStoragePolicyResponseProto { // void response } +message GetStoragePolicyRequestProto { + required string path = 1; +} + +message GetStoragePolicyResponseProto { + required BlockStoragePolicyProto storagePolicy = 1; +} + message GetStoragePoliciesRequestProto { // void request } @@ -725,6 +733,8 @@ service ClientNamenodeProtocol { returns(SetReplicationResponseProto); rpc setStoragePolicy(SetStoragePolicyRequestProto) returns(SetStoragePolicyResponseProto); + rpc getStoragePolicy(GetStoragePolicyRequestProto) + returns(GetStoragePolicyResponseProto); rpc getStoragePolicies(GetStoragePoliciesRequestProto) returns(GetStoragePoliciesResponseProto); rpc setPermission(SetPermissionRequestProto) http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc71ad80/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f093eea..40f91f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -770,6 +770,9 @@ Release 2.8.0 - UNRELEASED HDFS-6860. BlockStateChange logs are too noisy. (Chang Li and xyao via xyao) + HDFS-8815. DFS getStoragePolicy implementation using single RPC call + (Surendra Singh Lilhore via vinayakumarb) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc71ad80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 44713a4..3f4621e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1574,21 +1574,22 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } /** + * @param path file/directory name * @return Get the storage policy for specified path */ public BlockStoragePolicy getStoragePolicy(String path) throws IOException { - HdfsFileStatus status = getFileInfo(path); - if (status == null) { - throw new FileNotFoundException("File does not exist: " + path); - } - byte storagePolicyId = status.getStoragePolicy(); - BlockStoragePolicy[] policies = getStoragePolicies(); - for (BlockStoragePolicy policy : policies) { - if (policy.getId() == storagePolicyId) { - return policy; - } + checkOpen(); + TraceScope scope = getPathTraceScope("getStoragePolicy", path); + try { + return namenode.getStoragePolicy(path); + } catch (RemoteException e) { + throw e.unwrapRemoteException(AccessControlException.class, + FileNotFoundException.class, + SafeModeException.class, + UnresolvedPathException.class); + } finally { + scope.close(); } - return null; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc71ad80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 480b3d9..8e81fdc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -128,6 +128,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSna import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePolicyRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePolicyResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.IsFileClosedRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.IsFileClosedResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCacheDirectivesRequestProto; @@ -198,6 +200,7 @@ import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathR import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto; import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesResponseProto; import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockStoragePolicyProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; @@ -1458,6 +1461,20 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements } @Override + public GetStoragePolicyResponseProto getStoragePolicy( + RpcController controller, GetStoragePolicyRequestProto request) + throws ServiceException { + try { + BlockStoragePolicyProto policy = PBHelper.convert(server + .getStoragePolicy(request.getPath())); + return GetStoragePolicyResponseProto.newBuilder() + .setStoragePolicy(policy).build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override public GetStoragePoliciesResponseProto getStoragePolicies( RpcController controller, GetStoragePoliciesRequestProto request) throws ServiceException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc71ad80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 566d54f..d6afa6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -124,6 +124,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSna import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshottableDirListingResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePoliciesResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetStoragePolicyRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.IsFileClosedRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCacheDirectivesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ListCacheDirectivesResponseProto; @@ -1485,6 +1486,18 @@ public class ClientNamenodeProtocolTranslatorPB implements } @Override + public BlockStoragePolicy getStoragePolicy(String path) throws IOException { + GetStoragePolicyRequestProto request = GetStoragePolicyRequestProto + .newBuilder().setPath(path).build(); + try { + return PBHelper.convert(rpcProxy.getStoragePolicy(null, request) + .getStoragePolicy()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + + @Override public BlockStoragePolicy[] getStoragePolicies() throws IOException { try { GetStoragePoliciesResponseProto response = rpcProxy http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc71ad80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java index b322b69..d624f84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java @@ -200,6 +200,29 @@ public class FSDirAttrOp { return bm.getStoragePolicies(); } + static BlockStoragePolicy getStoragePolicy(FSDirectory fsd, BlockManager bm, + String path) throws IOException { + FSPermissionChecker pc = fsd.getPermissionChecker(); + byte[][] pathComponents = FSDirectory + .getPathComponentsForReservedPath(path); + fsd.readLock(); + try { + path = fsd.resolvePath(pc, path, pathComponents); + final INodesInPath iip = fsd.getINodesInPath(path, false); + if (fsd.isPermissionEnabled()) { + fsd.checkPathAccess(pc, iip, FsAction.READ); + } + INode inode = iip.getLastINode(); + if (inode == null) { + throw new FileNotFoundException("File/Directory does not exist: " + + iip.getPath()); + } + return bm.getStoragePolicy(inode.getStoragePolicyID()); + } finally { + fsd.readUnlock(); + } + } + static long getPreferredBlockSize(FSDirectory fsd, String src) throws IOException { FSPermissionChecker pc = fsd.getPermissionChecker(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc71ad80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index a259070..e3717ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1957,6 +1957,25 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, } /** + * Get the storage policy for a file or a directory. + * + * @param src + * file/directory path + * @return storage policy object + */ + BlockStoragePolicy getStoragePolicy(String src) throws IOException { + checkOperation(OperationCategory.READ); + waitForLoadingFSImage(); + readLock(); + try { + checkOperation(OperationCategory.READ); + return FSDirAttrOp.getStoragePolicy(dir, blockManager, src); + } finally { + readUnlock(); + } + } + + /** * @return All the existing block storage policies */ BlockStoragePolicy[] getStoragePolicies() throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc71ad80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 52aaabd..6b7e8cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -691,6 +691,12 @@ class NameNodeRpcServer implements NamenodeProtocols { } @Override + public BlockStoragePolicy getStoragePolicy(String path) throws IOException { + checkNNStartup(); + return namesystem.getStoragePolicy(path); + } + + @Override public BlockStoragePolicy[] getStoragePolicies() throws IOException { checkNNStartup(); return namesystem.getStoragePolicies(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc71ad80/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java index 631d9f7..689a1d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java @@ -980,6 +980,29 @@ public class TestBlockStoragePolicy { } @Test + public void testGetStoragePolicy() throws Exception { + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(REPLICATION).build(); + cluster.waitActive(); + final DistributedFileSystem fs = cluster.getFileSystem(); + try { + final Path dir = new Path("/testGetStoragePolicy"); + final Path fooFile = new Path(dir, "foo"); + DFSTestUtil.createFile(fs, fooFile, FILE_LEN, REPLICATION, 0L); + DFSClient client = new DFSClient(cluster.getNameNode(0) + .getNameNodeAddress(), conf); + client.setStoragePolicy("/testGetStoragePolicy/foo", + HdfsConstants.COLD_STORAGE_POLICY_NAME); + String policyName = client.getStoragePolicy("/testGetStoragePolicy/foo") + .getName(); + Assert.assertEquals("File storage policy should be COLD", + HdfsConstants.COLD_STORAGE_POLICY_NAME, policyName); + } finally { + cluster.shutdown(); + } + } + + @Test public void testSetStoragePolicyWithSnapshot() throws Exception { final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(REPLICATION).build();
