[
https://issues.apache.org/jira/browse/HDFS-15554?focusedWorklogId=477488&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-477488
]
ASF GitHub Bot logged work on HDFS-15554:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 01/Sep/20 22:08
Start Date: 01/Sep/20 22:08
Worklog Time Spent: 10m
Work Description: goiri commented on a change in pull request #2266:
URL: https://github.com/apache/hadoop/pull/2266#discussion_r481458397
##########
File path:
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
##########
@@ -128,7 +166,6 @@ public void testAddMountTable() throws IOException {
MountTable newEntry = MountTable.newInstance(
"/testpath", Collections.singletonMap("ns0", "/testdir"),
Time.now(), Time.now());
-
Review comment:
Avoid
##########
File path:
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
##########
@@ -103,11 +113,39 @@ public static void globalSetUp() throws Exception {
createNamenodeReport("ns1", "nn1", HAServiceState.ACTIVE));
stateStore.refreshCaches(true);
+ setUpMocks();
+ }
+
+ private static void setUpMocks() throws IOException {
RouterRpcServer spyRpcServer =
Mockito.spy(routerContext.getRouter().createRpcServer());
Whitebox
.setInternalState(routerContext.getRouter(), "rpcServer",
spyRpcServer);
Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString());
+
+ mockRpcClient = Mockito.spy(spyRpcServer.getRPCClient());
Review comment:
Add a comment highlighting what this part does.
##########
File path:
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
##########
@@ -103,11 +113,39 @@ public static void globalSetUp() throws Exception {
createNamenodeReport("ns1", "nn1", HAServiceState.ACTIVE));
stateStore.refreshCaches(true);
+ setUpMocks();
+ }
+
+ private static void setUpMocks() throws IOException {
RouterRpcServer spyRpcServer =
Mockito.spy(routerContext.getRouter().createRpcServer());
Whitebox
.setInternalState(routerContext.getRouter(), "rpcServer",
spyRpcServer);
Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString());
+
+ mockRpcClient = Mockito.spy(spyRpcServer.getRPCClient());
+ Whitebox
Review comment:
I think whitebox needed to be deprecated.
##########
File path:
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
##########
@@ -103,11 +113,39 @@ public static void globalSetUp() throws Exception {
createNamenodeReport("ns1", "nn1", HAServiceState.ACTIVE));
stateStore.refreshCaches(true);
+ setUpMocks();
+ }
+
+ private static void setUpMocks() throws IOException {
Review comment:
Add javadoc
##########
File path:
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
##########
@@ -103,11 +113,39 @@ public static void globalSetUp() throws Exception {
createNamenodeReport("ns1", "nn1", HAServiceState.ACTIVE));
stateStore.refreshCaches(true);
+ setUpMocks();
+ }
+
+ private static void setUpMocks() throws IOException {
RouterRpcServer spyRpcServer =
Mockito.spy(routerContext.getRouter().createRpcServer());
Whitebox
.setInternalState(routerContext.getRouter(), "rpcServer",
spyRpcServer);
Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString());
+
+ mockRpcClient = Mockito.spy(spyRpcServer.getRPCClient());
+ Whitebox
+ .setInternalState(spyRpcServer, "rpcClient", mockRpcClient);
+ RemoteLocation remoteLocation0 = new RemoteLocation("ns0", "/testdir",
null);
+ RemoteLocation remoteLocation1 = new RemoteLocation("ns1", "/", null);
+ mockResponse0.put(remoteLocation0,
+ new HdfsFileStatus.Builder().build());
+ Mockito.doReturn(mockResponse0).when(mockRpcClient).invokeConcurrent(
+ Mockito.eq(Lists.newArrayList(remoteLocation0)),
+ Mockito.any(RemoteMethod.class),
+ Mockito.eq(false),
+ Mockito.eq(false),
+ Mockito.eq(HdfsFileStatus.class)
+ );
+ mockResponse1.put(remoteLocation1,
+ new HdfsFileStatus.Builder().build());
+ Mockito.doReturn(mockResponse1).when(mockRpcClient).invokeConcurrent(
Review comment:
What are we doing with this? Is there a test actually using it?
##########
File path:
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
##########
@@ -562,11 +595,35 @@ public GetDestinationResponse getDestination(
LOG.error("Cannot get location for {}: {}",
src, ioe.getMessage());
}
- if (nsIds.isEmpty() && !locations.isEmpty()) {
- String nsId = locations.get(0).getNameserviceId();
- nsIds.add(nsId);
+ return nsIds;
+ }
+
+ /**
+ * Verify the file exists in destination nameservices to avoid dangling
+ * mount points.
+ *
+ * @param entry the new mount points added, could be from add or update.
+ * @return destination nameservices where the file doesn't exist.
+ * @throws IOException
Review comment:
Add exception reasons
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 477488)
Remaining Estimate: 0h
Time Spent: 10m
> RBF: force router check file existence before adding/updating mount points
> --------------------------------------------------------------------------
>
> Key: HDFS-15554
> URL: https://issues.apache.org/jira/browse/HDFS-15554
> Project: Hadoop HDFS
> Issue Type: Improvement
> Reporter: Fengnan Li
> Assignee: Fengnan Li
> Priority: Minor
> Time Spent: 10m
> Remaining Estimate: 0h
>
> Adding/Updating mount points right now is only a router action without
> validation in the downstream namenodes for the destination files/directories.
> In practice we have set up the dangling mount points and when clients call
> listStatus they would get the file returned, but then if they try to access
> the file FileNotFoundException would be thrown out.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
