Repository: hbase Updated Branches: refs/heads/branch-2.0 4932d8101 -> dc19f0b03
HBASE-20914 Trim Master memory usage Add (weak reference) interning of ServerNames. Correct Balancer regions x racks matrix. Make smaller defaults when creating ArrayDeques. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dc19f0b0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dc19f0b0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dc19f0b0 Branch: refs/heads/branch-2.0 Commit: dc19f0b03c4d617ecb5b407b623130d96235e697 Parents: 4932d81 Author: Michael Stack <[email protected]> Authored: Thu Jul 19 15:17:40 2018 -0700 Committer: Michael Stack <[email protected]> Committed: Fri Jul 20 10:07:42 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/ServerName.java | 17 ++++++++--- .../hadoop/hbase/procedure2/ProcedureDeque.java | 4 +++ .../hbase/master/balancer/BaseLoadBalancer.java | 2 +- .../master/balancer/StochasticLoadBalancer.java | 3 +- .../org/apache/hadoop/hbase/TestServerName.java | 30 ++++++++++++++++---- 5 files changed, 43 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/dc19f0b0/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java index f79b17a..34ac1e5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java @@ -27,9 +27,11 @@ import java.util.regex.Pattern; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.thirdparty.com.google.common.collect.Interner; +import org.apache.hbase.thirdparty.com.google.common.collect.Interners; +import org.apache.hbase.thirdparty.com.google.common.net.InetAddresses; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hbase.thirdparty.com.google.common.net.InetAddresses; /** @@ -99,6 +101,13 @@ public class ServerName implements Comparable<ServerName>, Serializable { private byte [] bytes; public static final List<ServerName> EMPTY_SERVER_LIST = new ArrayList<>(0); + /** + * Intern ServerNames. The Set of ServerNames is mostly-fixed changing slowly as Servers + * restart. Rather than create a new instance everytime, try and return existing instance + * if there is one. + */ + private static final Interner<ServerName> INTERN_POOL = Interners.newWeakInterner(); + protected ServerName(final String hostname, final int port, final long startcode) { this(Address.fromParts(hostname, port), startcode); } @@ -176,7 +185,7 @@ public class ServerName implements Comparable<ServerName>, Serializable { * a shared immutable object as an internal optimization. */ public static ServerName valueOf(final String hostname, final int port, final long startcode) { - return new ServerName(hostname, port, startcode); + return INTERN_POOL.intern(new ServerName(hostname, port, startcode)); } /** @@ -185,7 +194,7 @@ public class ServerName implements Comparable<ServerName>, Serializable { * a shared immutable object as an internal optimization. */ public static ServerName valueOf(final String serverName) { - return new ServerName(serverName); + return INTERN_POOL.intern(new ServerName(serverName)); } /** @@ -194,7 +203,7 @@ public class ServerName implements Comparable<ServerName>, Serializable { * a shared immutable object as an internal optimization. */ public static ServerName valueOf(final String hostAndPort, final long startCode) { - return new ServerName(hostAndPort, startCode); + return INTERN_POOL.intern(new ServerName(hostAndPort, startCode)); } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/dc19f0b0/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureDeque.java ---------------------------------------------------------------------- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureDeque.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureDeque.java index 41b8ca9..617e384 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureDeque.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureDeque.java @@ -31,4 +31,8 @@ import java.util.ArrayDeque; */ @InterfaceAudience.Private public class ProcedureDeque extends ArrayDeque<Procedure> { + public ProcedureDeque() { + // Default is 16 for a list that is rarely used; elements will resize if too small. + super(2); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/dc19f0b0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 98c3843..4c6ba99 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -566,7 +566,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer { * and rack have the highest locality for region */ private void computeCachedLocalities() { - rackLocalities = new float[numRegions][numServers]; + rackLocalities = new float[numRegions][numRacks]; regionsToMostLocalEntities = new int[LocalityType.values().length][numRegions]; // Compute localities and find most local server per region http://git-wip-us.apache.org/repos/asf/hbase/blob/dc19f0b0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index ac5ad64..ba080a3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -530,8 +530,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -> { Deque<BalancerRegionLoad> rLoads = oldLoads.get(Bytes.toString(regionName)); if (rLoads == null) { - // There was nothing there - rLoads = new ArrayDeque<>(); + rLoads = new ArrayDeque<>(numRegionLoadsToRemember + 1); } else if (rLoads.size() >= numRegionLoadsToRemember) { rLoads.remove(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/dc19f0b0/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerName.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerName.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerName.java index 6ed7c6b..fcbe9c9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerName.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerName.java @@ -20,16 +20,19 @@ package org.apache.hadoop.hbase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import java.util.HashSet; import java.util.Set; import java.util.regex.Pattern; + import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.Bytes; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -54,7 +57,7 @@ public class TestServerName { @Test public void testGetHostNameMinusDomain() { assertEquals("2607:f0d0:1002:51::4", - ServerName.getHostNameMinusDomain("2607:f0d0:1002:51::4")); + ServerName.getHostNameMinusDomain("2607:f0d0:1002:51::4")); assertEquals("2607:f0d0:1002:0051:0000:0000:0000:0004", ServerName.getHostNameMinusDomain("2607:f0d0:1002:0051:0000:0000:0000:0004")); assertEquals("1.1.1.1", ServerName.getHostNameMinusDomain("1.1.1.1")); @@ -86,10 +89,11 @@ public class TestServerName { ServerName.parseServerName("192.168.1.199:58102"); } - @Test public void testParseOfBytes() { + @Test + public void testParseOfBytes() { final String snStr = "www.EXAMPLE.org,1234,5678"; ServerName sn = ServerName.valueOf(snStr); - byte [] versionedBytes = sn.getVersionedBytes(); + byte[] versionedBytes = sn.getVersionedBytes(); ServerName parsedSn = ServerName.parseVersionedServerName(versionedBytes); assertEquals(sn.toString(), parsedSn.toString()); assertEquals(sn.getHostnameLowerCase(), parsedSn.getHostnameLowerCase()); @@ -97,7 +101,7 @@ public class TestServerName { assertEquals(sn.getStartcode(), parsedSn.getStartcode()); final String hostnamePortStr = sn.getAddress().toString(); - byte [] bytes = Bytes.toBytes(hostnamePortStr); + byte[] bytes = Bytes.toBytes(hostnamePortStr); parsedSn = ServerName.parseVersionedServerName(bytes); assertEquals(sn.getHostnameLowerCase(), parsedSn.getHostnameLowerCase()); assertEquals(sn.getPort(), parsedSn.getPort()); @@ -114,9 +118,9 @@ public class TestServerName { assertEquals(sn.hashCode(), sn2.hashCode()); assertNotSame(sn.hashCode(), sn3.hashCode()); assertEquals(sn.toString(), - ServerName.valueOf("www.example.org", 1234, 5678).toString()); + ServerName.valueOf("www.example.org", 1234, 5678).toString()); assertEquals(sn.toString(), - ServerName.valueOf("www.example.org:1234", 5678).toString()); + ServerName.valueOf("www.example.org:1234", 5678).toString()); assertEquals("www.example.org" + ServerName.SERVERNAME_SEPARATOR + "1234" + ServerName.SERVERNAME_SEPARATOR + "5678", sn.toString()); } @@ -132,5 +136,19 @@ public class TestServerName { assertTrue(upper.equals(lower)); assertTrue(ServerName.isSameAddress(lower, upper)); } + + @Test + public void testInterning() { + ServerName sn1 = ServerName.valueOf("www.example.org", 1234, 5671); + assertSame(sn1, ServerName.valueOf("www.example.org", 1234, 5671)); + } + + @Ignore // Enable and let fun for hours to make sure weak references working fine. + @Test + public void testInterningDoesWeakReferences() { + for (int i = 0; i < Integer.MAX_VALUE; i++) { + ServerName.valueOf("www.example.org", 1234, i++); + } + } }
