tasanuma commented on a change in pull request #3117:
URL: https://github.com/apache/hadoop/pull/3117#discussion_r656413524
##########
File path:
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedBlock.java
##########
@@ -126,11 +119,189 @@ public void testWithMultipleStateDatanodes() {
&& decommissionedNodes.contains(locations[4]));
}
- private static DatanodeManager mockDatanodeManager() throws IOException {
+ /**
+ * Test to verify sorting with multiple state
+ * datanodes exists in storage lists.
+ *
+ * After sorting the expected datanodes list will be:
+ * live -> slow -> stale -> staleAndSlow ->
+ * entering_maintenance -> decommissioned.
+ *
+ * avoidStaleDataNodesForRead=true && avoidSlowDataNodesForRead=true
+ * (d5 -> d4 -> d3 -> d2 -> d1 -> d0)
+ */
+ @Test(timeout = 30000)
+ public void testAviodStaleAndSlowDatanodes() throws IOException {
+ DatanodeManager dm = mockDatanodeManager(true, true);
+ DatanodeInfo[] locs = mockDatanodes(dm);
+
+ ArrayList<LocatedBlock> locatedBlocks = new ArrayList<>();
+ locatedBlocks.add(new LocatedBlock(
+ new ExtendedBlock("pool", Long.MIN_VALUE,
+ 1024L, new Date().getTime()), locs));
+
+ // sort located blocks
+ dm.sortLocatedBlocks(null, locatedBlocks);
+
+ // get locations after sorting
+ LocatedBlock locatedBlock = locatedBlocks.get(0);
+ DatanodeInfoWithStorage[] locations = locatedBlock.getLocations();
+
+ // assert location order:
+ // live -> stale -> entering_maintenance -> decommissioned
+ // live
+ assertEquals(locs[5].getIpAddr(), locations[0].getIpAddr());
+ // slow
+ assertEquals(locs[4].getIpAddr(), locations[1].getIpAddr());
+ // stale
+ assertEquals(locs[3].getIpAddr(), locations[2].getIpAddr());
+ // stale and slow
+ assertEquals(locs[2].getIpAddr(), locations[3].getIpAddr());
+ // entering_maintenance
+ assertEquals(locs[1].getIpAddr(), locations[4].getIpAddr());
+ // decommissioned
+ assertEquals(locs[0].getIpAddr(), locations[5].getIpAddr());
+ }
+
+ /**
+ * Test to verify sorting with multiple state
+ * datanodes exists in storage lists.
+ *
+ * After sorting the expected datanodes list will be:
+ * (live <-> slow) -> (stale <-> staleAndSlow) ->
+ * entering_maintenance -> decommissioned.
+ *
+ * avoidStaleDataNodesForRead=true && avoidSlowDataNodesForRead=false
+ * (d5 <-> d4) -> (d3 <-> d2) -> d1 -> d0
+ */
+ @Test(timeout = 30000)
+ public void testAviodStaleDatanodes() throws IOException {
+ DatanodeManager dm = mockDatanodeManager(true, false);
+ DatanodeInfo[] locs = mockDatanodes(dm);
+
+ ArrayList<LocatedBlock> locatedBlocks = new ArrayList<>();
+ locatedBlocks.add(new LocatedBlock(
+ new ExtendedBlock("pool", Long.MIN_VALUE,
+ 1024L, new Date().getTime()), locs));
+
+ // sort located blocks
+ dm.sortLocatedBlocks(null, locatedBlocks);
+
+ // get locations after sorting
+ LocatedBlock locatedBlock = locatedBlocks.get(0);
+ DatanodeInfoWithStorage[] locations = locatedBlock.getLocations();
+
+ // assert location order:
+ // live -> stale -> entering_maintenance -> decommissioned
+ // live
+ assertTrue((locs[5].getIpAddr() == locations[0].getIpAddr() &&
+ locs[4].getIpAddr() == locations[1].getIpAddr()) ||
+ (locs[5].getIpAddr() == locations[1].getIpAddr() &&
+ locs[4].getIpAddr() == locations[0].getIpAddr()));
+ // stale
+ assertTrue((locs[3].getIpAddr() == locations[2].getIpAddr() &&
+ locs[2].getIpAddr() == locations[3].getIpAddr()) ||
+ (locs[3].getIpAddr() == locations[3].getIpAddr() &&
+ locs[2].getIpAddr() == locations[2].getIpAddr()));
+ // entering_maintenance
+ assertEquals(locs[1].getIpAddr(), locations[4].getIpAddr());
+ // decommissioned
+ assertEquals(locs[0].getIpAddr(), locations[5].getIpAddr());
+ }
+
+ /**
+ * Test to verify sorting with multiple state
+ * datanodes exists in storage lists.
+ *
+ * After sorting the expected datanodes list will be:
+ * (live <-> stale) -> (slow <-> staleAndSlow) ->
+ * entering_maintenance -> decommissioned.
+ *
+ * avoidStaleDataNodesForRead=false && avoidSlowDataNodesForRead=true
+ * (d5 -> d3) -> ( d4 <-> d2) -> d1 -> d0)
Review comment:
```suggestion
* (d5 -> d3) -> (d4 <-> d2) -> d1 -> d0
```
##########
File path:
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedBlock.java
##########
@@ -126,11 +119,189 @@ public void testWithMultipleStateDatanodes() {
&& decommissionedNodes.contains(locations[4]));
}
- private static DatanodeManager mockDatanodeManager() throws IOException {
+ /**
+ * Test to verify sorting with multiple state
+ * datanodes exists in storage lists.
+ *
+ * After sorting the expected datanodes list will be:
+ * live -> slow -> stale -> staleAndSlow ->
+ * entering_maintenance -> decommissioned.
+ *
+ * avoidStaleDataNodesForRead=true && avoidSlowDataNodesForRead=true
+ * (d5 -> d4 -> d3 -> d2 -> d1 -> d0)
+ */
+ @Test(timeout = 30000)
+ public void testAviodStaleAndSlowDatanodes() throws IOException {
+ DatanodeManager dm = mockDatanodeManager(true, true);
+ DatanodeInfo[] locs = mockDatanodes(dm);
+
+ ArrayList<LocatedBlock> locatedBlocks = new ArrayList<>();
+ locatedBlocks.add(new LocatedBlock(
+ new ExtendedBlock("pool", Long.MIN_VALUE,
+ 1024L, new Date().getTime()), locs));
+
+ // sort located blocks
+ dm.sortLocatedBlocks(null, locatedBlocks);
+
+ // get locations after sorting
+ LocatedBlock locatedBlock = locatedBlocks.get(0);
+ DatanodeInfoWithStorage[] locations = locatedBlock.getLocations();
+
+ // assert location order:
+ // live -> stale -> entering_maintenance -> decommissioned
+ // live
+ assertEquals(locs[5].getIpAddr(), locations[0].getIpAddr());
+ // slow
+ assertEquals(locs[4].getIpAddr(), locations[1].getIpAddr());
+ // stale
+ assertEquals(locs[3].getIpAddr(), locations[2].getIpAddr());
+ // stale and slow
+ assertEquals(locs[2].getIpAddr(), locations[3].getIpAddr());
+ // entering_maintenance
+ assertEquals(locs[1].getIpAddr(), locations[4].getIpAddr());
+ // decommissioned
+ assertEquals(locs[0].getIpAddr(), locations[5].getIpAddr());
+ }
+
+ /**
+ * Test to verify sorting with multiple state
+ * datanodes exists in storage lists.
+ *
+ * After sorting the expected datanodes list will be:
+ * (live <-> slow) -> (stale <-> staleAndSlow) ->
+ * entering_maintenance -> decommissioned.
+ *
+ * avoidStaleDataNodesForRead=true && avoidSlowDataNodesForRead=false
+ * (d5 <-> d4) -> (d3 <-> d2) -> d1 -> d0
+ */
+ @Test(timeout = 30000)
+ public void testAviodStaleDatanodes() throws IOException {
+ DatanodeManager dm = mockDatanodeManager(true, false);
+ DatanodeInfo[] locs = mockDatanodes(dm);
+
+ ArrayList<LocatedBlock> locatedBlocks = new ArrayList<>();
+ locatedBlocks.add(new LocatedBlock(
+ new ExtendedBlock("pool", Long.MIN_VALUE,
+ 1024L, new Date().getTime()), locs));
+
+ // sort located blocks
+ dm.sortLocatedBlocks(null, locatedBlocks);
+
+ // get locations after sorting
+ LocatedBlock locatedBlock = locatedBlocks.get(0);
+ DatanodeInfoWithStorage[] locations = locatedBlock.getLocations();
+
+ // assert location order:
+ // live -> stale -> entering_maintenance -> decommissioned
+ // live
+ assertTrue((locs[5].getIpAddr() == locations[0].getIpAddr() &&
+ locs[4].getIpAddr() == locations[1].getIpAddr()) ||
+ (locs[5].getIpAddr() == locations[1].getIpAddr() &&
+ locs[4].getIpAddr() == locations[0].getIpAddr()));
+ // stale
+ assertTrue((locs[3].getIpAddr() == locations[2].getIpAddr() &&
+ locs[2].getIpAddr() == locations[3].getIpAddr()) ||
+ (locs[3].getIpAddr() == locations[3].getIpAddr() &&
+ locs[2].getIpAddr() == locations[2].getIpAddr()));
+ // entering_maintenance
+ assertEquals(locs[1].getIpAddr(), locations[4].getIpAddr());
+ // decommissioned
+ assertEquals(locs[0].getIpAddr(), locations[5].getIpAddr());
+ }
+
+ /**
+ * Test to verify sorting with multiple state
+ * datanodes exists in storage lists.
+ *
+ * After sorting the expected datanodes list will be:
+ * (live <-> stale) -> (slow <-> staleAndSlow) ->
+ * entering_maintenance -> decommissioned.
+ *
+ * avoidStaleDataNodesForRead=false && avoidSlowDataNodesForRead=true
+ * (d5 -> d3) -> ( d4 <-> d2) -> d1 -> d0)
+ */
+ @Test(timeout = 30000)
+ public void testAviodSlowDatanodes() throws IOException {
+ DatanodeManager dm = mockDatanodeManager(false, true);
+ DatanodeInfo[] locs = mockDatanodes(dm);
+
+ ArrayList<LocatedBlock> locatedBlocks = new ArrayList<>();
+ locatedBlocks.add(new LocatedBlock(
+ new ExtendedBlock("pool", Long.MIN_VALUE,
+ 1024L, new Date().getTime()), locs));
+
+ // sort located blocks
+ dm.sortLocatedBlocks(null, locatedBlocks);
+
+ // get locations after sorting
+ LocatedBlock locatedBlock = locatedBlocks.get(0);
+ DatanodeInfoWithStorage[] locations = locatedBlock.getLocations();
+
+ // assert location order:
+ // live -> slow -> entering_maintenance -> decommissioned
+ // live
+ assertTrue((locs[5].getIpAddr() == locations[0].getIpAddr() &&
+ locs[3].getIpAddr() == locations[1].getIpAddr()) ||
+ (locs[5].getIpAddr() == locations[1].getIpAddr() &&
+ locs[3].getIpAddr() == locations[0].getIpAddr()));
+ // slow
+ assertTrue((locs[4].getIpAddr() == locations[2].getIpAddr() &&
+ locs[2].getIpAddr() == locations[3].getIpAddr()) ||
+ (locs[4].getIpAddr() == locations[3].getIpAddr() &&
+ locs[2].getIpAddr() == locations[2].getIpAddr()));
+ // entering_maintenance
+ assertEquals(locs[1].getIpAddr(), locations[4].getIpAddr());
+ // decommissioned
+ assertEquals(locs[0].getIpAddr(), locations[5].getIpAddr());
+ }
+
+ /**
+ * We mock the following list of datanodes, and create LocatedBlock.
+ * d0 - decommissioned
+ * d1 - entering_maintenance
+ * d2 - stale and slow
+ * d3 - stale
+ * d4 - slow
+ * d5 - live(in-service)
+ */
+ private static DatanodeInfo[] mockDatanodes(DatanodeManager dm)
+ throws IOException {
Review comment:
```suggestion
private static DatanodeInfo[] mockDatanodes(DatanodeManager dm) {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
