This is an automated email from the ASF dual-hosted git repository. xkrogen pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push: new 792e2ae HDFS-14979 Allow Balancer to submit getBlocks calls to Observer Nodes when possible. Contributed by Erik Krogen. 792e2ae is described below commit 792e2aecb3d81ce478f2a44350e4a64351967e52 Author: Erik Krogen <xkro...@apache.org> AuthorDate: Mon Nov 11 14:32:51 2019 -0800 HDFS-14979 Allow Balancer to submit getBlocks calls to Observer Nodes when possible. Contributed by Erik Krogen. (cherry picked from 586defe7113ed246ed0275bb3833882a3d873d70) (cherry picked from dec765b329d3947f30273c0e7f0c4eb607ec42c9) (cherry picked from 5cf36aa2b19aad8677e1a7553ff1f05805f772b7) --- .../hdfs/server/protocol/NamenodeProtocol.java | 2 ++ .../balancer/TestBalancerWithHANameNodes.java | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java index ccdf516c..f218e8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature; +import org.apache.hadoop.hdfs.server.namenode.ha.ReadOnly; import org.apache.hadoop.io.retry.AtMostOnce; import org.apache.hadoop.io.retry.Idempotent; import org.apache.hadoop.security.KerberosInfo; @@ -76,6 +77,7 @@ public interface NamenodeProtocol { datanode does not exist */ @Idempotent + @ReadOnly public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size) throws IOException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java index 4a398db..779e19c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java @@ -18,8 +18,14 @@ package org.apache.hadoop.hdfs.server.balancer; import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import java.net.URI; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -32,9 +38,13 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf; import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider; +import org.mockito.Matchers; import org.junit.Test; /** @@ -128,12 +138,24 @@ public class TestBalancerWithHANameNodes { cluster = qjmhaCluster.getDfsCluster(); cluster.waitClusterUp(); cluster.waitActive(); + List<FSNamesystem> namesystemSpies = new ArrayList<>(); + for (int i = 0; i < cluster.getNumNameNodes(); i++) { + namesystemSpies.add( + NameNodeAdapter.spyOnNamesystem(cluster.getNameNode(i))); + } DistributedFileSystem dfs = HATestUtil.configureObserverReadFs( cluster, conf, ObserverReadProxyProvider.class, true); client = dfs.getClient().getNamenode(); doTest(conf); + for (int i = 0; i < cluster.getNumNameNodes(); i++) { + // First observer node is at idx 2 so it should get both getBlocks calls + // all other NameNodes should see 0 getBlocks calls + int expectedCount = (i == 2) ? 2 : 0; + verify(namesystemSpies.get(i), times(expectedCount)) + .getBlocks(Matchers.<DatanodeID>any(), anyLong()); + } } finally { if (qjmhaCluster != null) { qjmhaCluster.shutdown(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org