[
https://issues.apache.org/jira/browse/HDFS-17597?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17873205#comment-17873205
]
ASF GitHub Bot commented on HDFS-17597:
---------------------------------------
KeeProMise commented on code in PR #6994:
URL: https://github.com/apache/hadoop/pull/6994#discussion_r1715335670
##########
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAsyncSnapshot.java:
##########
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
+import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import
org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import
org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.router.async.ApplyFunction;
+import org.apache.hadoop.hdfs.server.federation.router.async.AsyncUtil;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static
org.apache.hadoop.hdfs.server.federation.router.async.AsyncUtil.asyncApply;
+import static
org.apache.hadoop.hdfs.server.federation.router.async.AsyncUtil.asyncReturn;
+
+/**
+ * Module that implements all the asynchronous RPC calls related to snapshots
in
+ * {@link ClientProtocol} in the {@link RouterRpcServer}.
+ */
+public class RouterAsyncSnapshot extends RouterSnapshot {
+ /** RPC server to receive client calls. */
+ private final RouterRpcServer rpcServer;
+ /** RPC clients to connect to the Namenodes. */
+ private final RouterRpcClient rpcClient;
+ /** Find generic locations. */
+ private final ActiveNamenodeResolver namenodeResolver;
+
+ public RouterAsyncSnapshot(RouterRpcServer server) {
+ super(server);
+ this.rpcServer = server;
+ this.rpcClient = this.rpcServer.getRPCClient();
+ this.namenodeResolver = rpcServer.getNamenodeResolver();
+ }
+
+ @Override
+ public String createSnapshot(String snapshotRoot, String snapshotName)
throws IOException {
+ rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations =
+ rpcServer.getLocationsForPath(snapshotRoot, true, false);
+ RemoteMethod method = new RemoteMethod("createSnapshot",
+ new Class<?>[] {String.class, String.class}, new RemoteParam(),
+ snapshotName);
+
+ if (rpcServer.isInvokeConcurrent(snapshotRoot)) {
+ rpcClient.invokeConcurrent(locations, method, String.class);
+ asyncApply((ApplyFunction<Map<RemoteLocation, String>, String>)
+ results -> {
+ Map.Entry<RemoteLocation, String> firstelement =
+ results.entrySet().iterator().next();
+ RemoteLocation loc = firstelement.getKey();
+ String result = firstelement.getValue();
+ return result.replaceFirst(loc.getDest(), loc.getSrc());
+ });
+ } else {
+ rpcClient.invokeSequential(method, locations, String.class, null);
+ asyncApply((ApplyFunction<RemoteResult<RemoteLocation, String>, String>)
+ response -> {
+ RemoteLocation loc = response.getLocation();
+ String invokedResult = response.getResult();
+ return invokedResult.replaceFirst(loc.getDest(), loc.getSrc());
+ });
+ }
+ return AsyncUtil.asyncReturn(String.class);
Review Comment:
@LeoLeeeeee Hi, thank you for your contribution. I think AsyncUtil can be
deleted here. You can use asyncReturn(String.class) directly.
##########
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAsyncSnapshot.java:
##########
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
+import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import
org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import
org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.router.async.ApplyFunction;
+import org.apache.hadoop.hdfs.server.federation.router.async.AsyncUtil;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static
org.apache.hadoop.hdfs.server.federation.router.async.AsyncUtil.asyncApply;
+import static
org.apache.hadoop.hdfs.server.federation.router.async.AsyncUtil.asyncReturn;
+
+/**
+ * Module that implements all the asynchronous RPC calls related to snapshots
in
+ * {@link ClientProtocol} in the {@link RouterRpcServer}.
+ */
+public class RouterAsyncSnapshot extends RouterSnapshot {
+ /** RPC server to receive client calls. */
+ private final RouterRpcServer rpcServer;
+ /** RPC clients to connect to the Namenodes. */
+ private final RouterRpcClient rpcClient;
+ /** Find generic locations. */
+ private final ActiveNamenodeResolver namenodeResolver;
+
+ public RouterAsyncSnapshot(RouterRpcServer server) {
+ super(server);
+ this.rpcServer = server;
+ this.rpcClient = this.rpcServer.getRPCClient();
+ this.namenodeResolver = rpcServer.getNamenodeResolver();
+ }
+
+ @Override
+ public String createSnapshot(String snapshotRoot, String snapshotName)
throws IOException {
+ rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
+
+ final List<RemoteLocation> locations =
+ rpcServer.getLocationsForPath(snapshotRoot, true, false);
+ RemoteMethod method = new RemoteMethod("createSnapshot",
+ new Class<?>[] {String.class, String.class}, new RemoteParam(),
+ snapshotName);
+
+ if (rpcServer.isInvokeConcurrent(snapshotRoot)) {
+ rpcClient.invokeConcurrent(locations, method, String.class);
+ asyncApply((ApplyFunction<Map<RemoteLocation, String>, String>)
+ results -> {
+ Map.Entry<RemoteLocation, String> firstelement =
+ results.entrySet().iterator().next();
+ RemoteLocation loc = firstelement.getKey();
+ String result = firstelement.getValue();
+ return result.replaceFirst(loc.getDest(), loc.getSrc());
+ });
+ } else {
+ rpcClient.invokeSequential(method, locations, String.class, null);
+ asyncApply((ApplyFunction<RemoteResult<RemoteLocation, String>, String>)
+ response -> {
+ RemoteLocation loc = response.getLocation();
+ String invokedResult = response.getResult();
+ return invokedResult.replaceFirst(loc.getDest(), loc.getSrc());
+ });
+ }
+ return AsyncUtil.asyncReturn(String.class);
+ }
+
+ @Override
+ public SnapshottableDirectoryStatus[] getSnapshottableDirListing() throws
IOException {
+ rpcServer.checkOperation(NameNode.OperationCategory.READ);
+
+ RemoteMethod method = new RemoteMethod("getSnapshottableDirListing");
+ Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+ rpcClient.invokeConcurrent(
+ nss, method, true, false, SnapshottableDirectoryStatus[].class);
+ asyncApply((ApplyFunction<Map<FederationNamespaceInfo,
SnapshottableDirectoryStatus[]>,
+ SnapshottableDirectoryStatus[]>)
+ ret -> RouterRpcServer.merge(ret, SnapshottableDirectoryStatus.class));
+ return AsyncUtil.asyncReturn(SnapshottableDirectoryStatus[].class);
Review Comment:
Same as above.
> [ARR] RouterSnapshot supports asynchronous rpc.
> -----------------------------------------------
>
> Key: HDFS-17597
> URL: https://issues.apache.org/jira/browse/HDFS-17597
> Project: Hadoop HDFS
> Issue Type: Sub-task
> Reporter: Jian Zhang
> Assignee: farmmamba
> Priority: Major
> Labels: pull-request-available
>
> *Describe*
> The main new addition is RouterAsyncSnapshot, which extends RouterSnapshot so
> that supports asynchronous rpc.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
