This is an automated email from the ASF dual-hosted git repository. cliang pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 0f811b0849a7621a568bdaf2f0bb5eefeb62e0a9 Author: Erik Krogen <xkro...@apache.org> AuthorDate: Wed Aug 1 09:58:04 2018 -0700 HDFS-13688. [SBN read] Introduce msync API call. Contributed by Chen Liang. --- .../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 14 ++++++++++++++ .../org/apache/hadoop/hdfs/protocol/ClientProtocol.java | 11 +++++++++++ .../protocolPB/ClientNamenodeProtocolTranslatorPB.java | 11 +++++++++++ .../src/main/proto/ClientNamenodeProtocol.proto | 8 ++++++++ .../java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java | 3 ++- .../hdfs/server/federation/router/RouterRpcServer.java | 5 +++++ .../ClientNamenodeProtocolServerSideTranslatorPB.java | 13 +++++++++++++ .../hadoop/hdfs/server/namenode/NameNodeRpcServer.java | 5 +++++ 8 files changed, 69 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index eb77db6..4f708a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -3042,4 +3042,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, checkOpen(); return new OpenFilesIterator(namenode, tracer, openFilesTypes, path); } + + /** + * A blocking call to wait for Observer NameNode state ID to reach to the + * current client state ID. Current client state ID is given by the client + * alignment context. + * An assumption is that client alignment context has the state ID set at this + * point. This is become ObserverReadProxyProvider sets up the initial state + * ID when it is being created. + * + * @throws IOException + */ + public void msync() throws IOException { + namenode.msync(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 7923113..6196500 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1734,4 +1734,15 @@ public interface ClientProtocol { @ReadOnly BatchedEntries<OpenFileEntry> listOpenFiles(long prevId, EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException; + + /** + * Called by client to wait until the server has reached the state id of the + * client. The client and server state id are given by client side and server + * side alignment context respectively. This can be a blocking call. + * + * @throws IOException + */ + @Idempotent + @ReadOnly + void msync() throws IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index b843e29..60b98fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -152,6 +152,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.MetaSa import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.MkdirsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ModifyCacheDirectiveRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ModifyCachePoolRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.MsyncRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.MsyncResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.OpenFilesBatchResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RecoverLeaseRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RefreshNodesRequestProto; @@ -1880,4 +1882,13 @@ public class ClientNamenodeProtocolTranslatorPB implements } } + @Override + public void msync() throws IOException { + MsyncRequestProto.Builder req = MsyncRequestProto.newBuilder(); + try { + rpcProxy.msync(null, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index fad814e..a5aa082 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -802,6 +802,12 @@ message ListOpenFilesResponseProto { repeated OpenFilesTypeProto types = 3; } +message MsyncRequestProto { +} + +message MsyncResponseProto { +} + service ClientNamenodeProtocol { rpc getBlockLocations(GetBlockLocationsRequestProto) returns(GetBlockLocationsResponseProto); @@ -982,4 +988,6 @@ service ClientNamenodeProtocol { returns(GetQuotaUsageResponseProto); rpc listOpenFiles(ListOpenFilesRequestProto) returns(ListOpenFilesResponseProto); + rpc msync(MsyncRequestProto) + returns(MsyncResponseProto); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java index 34e84fa..57db8ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java @@ -71,7 +71,8 @@ public class TestReadOnly { "getDataEncryptionKey", "getCurrentEditLogTxid", "getEditsFromTxid", - "getQuotaUsage" + "getQuotaUsage", + "msync" ) ); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 47370b7..47dfc46 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -2071,6 +2071,11 @@ public class RouterRpcServer extends AbstractService return null; } + @Override // ClientProtocol + public void msync() throws IOException { + + } + @Override // NamenodeProtocol public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 889489f..d5f7b9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -168,6 +168,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Modify import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ModifyCacheDirectiveResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ModifyCachePoolRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ModifyCachePoolResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.MsyncRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.MsyncResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RecoverLeaseRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RecoverLeaseResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RefreshNodesRequestProto; @@ -1825,4 +1827,15 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements throw new ServiceException(e); } } + + @Override + public MsyncResponseProto msync(RpcController controller, + MsyncRequestProto req) throws ServiceException { + try { + server.msync(); + return MsyncResponseProto.newBuilder().build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index b0cb0a4..5d8e12b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -1341,6 +1341,11 @@ public class NameNodeRpcServer implements NamenodeProtocols { } @Override // ClientProtocol + public void msync() throws IOException { + // TODO : need to be filled up if needed. May be a no-op here. + } + + @Override // ClientProtocol public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie) throws IOException { checkNNStartup(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org