Repository: hbase Updated Branches: refs/heads/master 15831cefd -> 9f8d1876a
HBASE-11344 Hide row keys and such from the web UIs Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9f8d1876 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9f8d1876 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9f8d1876 Branch: refs/heads/master Commit: 9f8d1876a0412af2d751241cde511600cf7ac164 Parents: 15831ce Author: Devaraj Das <[email protected]> Authored: Wed Jul 2 13:07:11 2014 -0700 Committer: Devaraj Das <[email protected]> Committed: Wed Jul 2 13:07:11 2014 -0700 ---------------------------------------------------------------------- dev-support/findbugs-exclude.xml | 9 ++ .../org/apache/hadoop/hbase/HRegionInfo.java | 103 +++++++++++++++++++ .../src/main/resources/hbase-default.xml | 7 ++ .../master/AssignmentManagerStatusTmpl.jamon | 4 +- .../tmpl/regionserver/RegionListTmpl.jamon | 21 ++-- .../resources/hbase-webapps/master/table.jsp | 9 +- .../hbase/regionserver/TestHRegionInfo.java | 68 ++++++++++++ 7 files changed, 210 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/9f8d1876/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/dev-support/findbugs-exclude.xml b/dev-support/findbugs-exclude.xml index b2a609a..d89f9b2 100644 --- a/dev-support/findbugs-exclude.xml +++ b/dev-support/findbugs-exclude.xml @@ -261,4 +261,13 @@ <Bug pattern="FE_FLOATING_POINT_EQUALITY"/> </Match> + <Match> + <Class name="org.apache.hadoop.hbase.HRegionInfo"/> + <Or> + <Method name="getEndKeyForDisplay"/> + <Method name="getStartKeyForDisplay"/> + </Or> + <Bug pattern="MS_EXPOSE_REP"/> + </Match> + </FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hbase/blob/9f8d1876/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index 3e5224b..0799fc8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -33,9 +33,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo; @@ -220,6 +222,9 @@ public class HRegionInfo implements Comparable<HRegionInfo> { // Current TableName private TableName tableName = null; + final static String DISPLAY_KEYS_KEY = "hbase.display.keys"; + public final static byte[] HIDDEN_END_KEY = Bytes.toBytes("hidden-end-key"); + public final static byte[] HIDDEN_START_KEY = Bytes.toBytes("hidden-start-key"); /** HRegionInfo for first meta region */ public static final HRegionInfo FIRST_META_REGIONINFO = @@ -1123,6 +1128,104 @@ public class HRegionInfo implements Comparable<HRegionInfo> { } /** + * Get the descriptive name as {@link RegionState} does it but with hidden + * startkey optionally + * @param state + * @param conf + * @return descriptive string + */ + public static String getDescriptiveNameFromRegionStateForDisplay(RegionState state, + Configuration conf) { + if (conf.getBoolean(DISPLAY_KEYS_KEY, true)) return state.toDescriptiveString(); + String descriptiveStringFromState = state.toDescriptiveString(); + int idx = descriptiveStringFromState.lastIndexOf(" state="); + String regionName = getRegionNameAsStringForDisplay(state.getRegion(), conf); + return regionName + descriptiveStringFromState.substring(idx); + } + + /** + * Get the end key for display. Optionally hide the real end key. + * @param hri + * @param conf + * @return the endkey + */ + public static byte[] getEndKeyForDisplay(HRegionInfo hri, Configuration conf) { + boolean displayKey = conf.getBoolean(DISPLAY_KEYS_KEY, true); + if (displayKey) return hri.getEndKey(); + return HIDDEN_END_KEY; + } + + /** + * Get the start key for display. Optionally hide the real start key. + * @param hri + * @param conf + * @return the startkey + */ + public static byte[] getStartKeyForDisplay(HRegionInfo hri, Configuration conf) { + boolean displayKey = conf.getBoolean(DISPLAY_KEYS_KEY, true); + if (displayKey) return hri.getStartKey(); + return HIDDEN_START_KEY; + } + + /** + * Get the region name for display. Optionally hide the start key. + * @param hri + * @param conf + * @return region name as String + */ + public static String getRegionNameAsStringForDisplay(HRegionInfo hri, Configuration conf) { + return Bytes.toStringBinary(getRegionNameForDisplay(hri, conf)); + } + + /** + * Get the region name for display. Optionally hide the start key. + * @param hri + * @param conf + * @return region name bytes + */ + public static byte[] getRegionNameForDisplay(HRegionInfo hri, Configuration conf) { + boolean displayKey = conf.getBoolean(DISPLAY_KEYS_KEY, true); + if (displayKey || hri.getTable().equals(TableName.META_TABLE_NAME)) { + return hri.getRegionName(); + } else { + // create a modified regionname with the startkey replaced but preserving + // the other parts including the encodedname. + try { + byte[][]regionNameParts = parseRegionName(hri.getRegionName()); + regionNameParts[1] = HIDDEN_START_KEY; //replace the real startkey + int len = 0; + // get the total length + for (byte[] b : regionNameParts) { + len += b.length; + } + byte[] encodedRegionName = + Bytes.toBytes(encodeRegionName(hri.getRegionName())); + len += encodedRegionName.length; + //allocate some extra bytes for the delimiters and the last '.' + byte[] modifiedName = new byte[len + regionNameParts.length + 1]; + int lengthSoFar = 0; + int loopCount = 0; + for (byte[] b : regionNameParts) { + System.arraycopy(b, 0, modifiedName, lengthSoFar, b.length); + lengthSoFar += b.length; + if (loopCount++ == 2) modifiedName[lengthSoFar++] = REPLICA_ID_DELIMITER; + else modifiedName[lengthSoFar++] = HConstants.DELIMITER; + } + // replace the last comma with '.' + modifiedName[lengthSoFar - 1] = ENC_SEPARATOR; + System.arraycopy(encodedRegionName, 0, modifiedName, lengthSoFar, + encodedRegionName.length); + lengthSoFar += encodedRegionName.length; + modifiedName[lengthSoFar] = ENC_SEPARATOR; + return modifiedName; + } catch (IOException e) { + //LOG.warn("Encountered exception " + e); + throw new RuntimeException(e); + } + } + } + + /** * Extract a HRegionInfo and ServerName from catalog table {@link Result}. * @param r Result to pull from * @return A pair of the {@link HRegionInfo} and the {@link ServerName} http://git-wip-us.apache.org/repos/asf/hbase/blob/9f8d1876/hbase-common/src/main/resources/hbase-default.xml ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 70d20a7..33dd6f6 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -837,6 +837,13 @@ possible configurations would overwhelm and obscure the important. authentication, and will abort the connection.</description> </property> <property> + <name>hbase.display.keys</name> + <value>true</value> + <description>When this is set to true the webUI and such will display all start/end keys + as part of the table details, region names, etc. When this is set to false, + the keys are hidden.</description> + </property> + <property> <name>hbase.coprocessor.region.classes</name> <value></value> <description>A comma-separated list of Coprocessors that are loaded by http://git-wip-us.apache.org/repos/asf/hbase/blob/9f8d1876/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon index 08ed672..f6ea464 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon @@ -85,7 +85,9 @@ if (toRemove > 0) { <%else> <tr> </%if> - <td><% entry.getKey() %></td><td><% entry.getValue().toDescriptiveString() %></td> + <td><% entry.getKey() %></td><td> + <% HRegionInfo.getDescriptiveNameFromRegionStateForDisplay( + entry.getValue(), conf) %></td> <td><% (currentTime - entry.getValue().getStamp()) %> </td></tr> </%for> <tr BGCOLOR="#D7DF01"> <td>Total number of Regions in Transition for more than <% ritThreshold %> milliseconds</td><td> <% numOfRITOverThreshold %></td><td></td> http://git-wip-us.apache.org/repos/asf/hbase/blob/9f8d1876/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon index 8606b4e..7c4f1f1 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon @@ -93,9 +93,12 @@ <%for HRegionInfo r: onlineRegions %> <tr> - <td><% r.getRegionNameAsString() %></td> - <td><% Bytes.toStringBinary(r.getStartKey()) %></td> - <td><% Bytes.toStringBinary(r.getEndKey()) %></td> + <td><% HRegionInfo.getRegionNameAsStringForDisplay(r, + regionServer.getConfiguration()) %></td> + <td><% Bytes.toStringBinary(HRegionInfo.getStartKeyForDisplay(r, + regionServer.getConfiguration())) %></td> + <td><% Bytes.toStringBinary(HRegionInfo.getEndKeyForDisplay(r, + regionServer.getConfiguration())) %></td> <td><% r.getReplicaId() %></td> </tr> </%for> @@ -119,7 +122,8 @@ <%java> RegionLoad load = regionServer.createRegionLoad(r.getEncodedName()); </%java> - <td><% r.getRegionNameAsString() %></td> + <td><% HRegionInfo.getRegionNameAsStringForDisplay(r, + regionServer.getConfiguration()) %></td> <%if load != null %> <td><% load.getReadRequestsCount() %></td> <td><% load.getWriteRequestsCount() %></td> @@ -151,7 +155,8 @@ <%java> RegionLoad load = regionServer.createRegionLoad(r.getEncodedName()); </%java> - <td><% r.getRegionNameAsString() %></td> + <td><% HRegionInfo.getRegionNameAsStringForDisplay(r, + regionServer.getConfiguration()) %></td> <%if load != null %> <td><% load.getStores() %></td> <td><% load.getStorefiles() %></td> @@ -189,7 +194,8 @@ ((float) load.getCurrentCompactedKVs() / load.getTotalCompactingKVs())) + "%"; } </%java> - <td><% r.getRegionNameAsString() %></td> + <td><% HRegionInfo.getRegionNameAsStringForDisplay(r, + regionServer.getConfiguration()) %></td> <%if load != null %> <td><% load.getTotalCompactingKVs() %></td> <td><% load.getCurrentCompactedKVs() %></td> @@ -216,7 +222,8 @@ <%java> RegionLoad load = regionServer.createRegionLoad(r.getEncodedName()); </%java> - <td><% r.getRegionNameAsString() %></td> + <td><% HRegionInfo.getRegionNameAsStringForDisplay(r, + regionServer.getConfiguration()) %></td> <%if load != null %> <td><% load.getMemstoreSizeMB() %>m</td> </%if> http://git-wip-us.apache.org/repos/asf/hbase/blob/9f8d1876/hbase-server/src/main/resources/hbase-webapps/master/table.jsp ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 4fa0299..3107226 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -283,7 +283,8 @@ } %> <tr> - <td><%= escapeXml(Bytes.toStringBinary(regionInfo.getRegionName())) %></td> + <td><%= escapeXml(Bytes.toStringBinary(HRegionInfo.getRegionNameForDisplay(regionInfo, + conf))) %></td> <% if (addr != null) { String url = "//" + addr.getHostname() + ":" + master.getRegionServerInfoPort(addr) + "/"; @@ -298,8 +299,10 @@ <% } %> - <td><%= escapeXml(Bytes.toStringBinary(regionInfo.getStartKey())) %></td> - <td><%= escapeXml(Bytes.toStringBinary(regionInfo.getEndKey())) %></td> + <td><%= escapeXml(Bytes.toStringBinary(HRegionInfo.getStartKeyForDisplay(regionInfo, + conf))) %></td> + <td><%= escapeXml(Bytes.toStringBinary(HRegionInfo.getEndKeyForDisplay(regionInfo, + conf))) %></td> <td><%= req%></td> <% if (withReplica) { http://git-wip-us.apache.org/repos/asf/hbase/blob/9f8d1876/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java index db4e5cf..a9aa456 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java @@ -26,6 +26,7 @@ import static org.junit.Assert.fail; import java.io.IOException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -34,10 +35,12 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.MD5Hash; +import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -254,6 +257,71 @@ public class TestHRegionInfo { assertEquals(expectedHri, convertedHri); } + @Test + public void testRegionDetailsForDisplay() throws IOException { + byte[] startKey = new byte[] {0x01, 0x01, 0x02, 0x03}; + byte[] endKey = new byte[] {0x01, 0x01, 0x02, 0x04}; + Configuration conf = new Configuration(); + conf.setBoolean("hbase.display.keys", false); + HRegionInfo h = new HRegionInfo(TableName.valueOf("foo"), startKey, endKey); + checkEquality(h, conf); + // check HRIs with non-default replicaId + h = new HRegionInfo(TableName.valueOf("foo"), startKey, endKey, false, + System.currentTimeMillis(), 1); + checkEquality(h, conf); + Assert.assertArrayEquals(HRegionInfo.HIDDEN_END_KEY, + HRegionInfo.getEndKeyForDisplay(h, conf)); + Assert.assertArrayEquals(HRegionInfo.HIDDEN_START_KEY, + HRegionInfo.getStartKeyForDisplay(h, conf)); + + RegionState state = new RegionState(h, RegionState.State.OPEN); + String descriptiveNameForDisplay = + HRegionInfo.getDescriptiveNameFromRegionStateForDisplay(state, conf); + checkDescriptiveNameEquality(descriptiveNameForDisplay,state.toDescriptiveString(), startKey); + + conf.setBoolean("hbase.display.keys", true); + Assert.assertArrayEquals(endKey, HRegionInfo.getEndKeyForDisplay(h, conf)); + Assert.assertArrayEquals(startKey, HRegionInfo.getStartKeyForDisplay(h, conf)); + Assert.assertEquals(state.toDescriptiveString(), + HRegionInfo.getDescriptiveNameFromRegionStateForDisplay(state, conf)); + } + + private void checkDescriptiveNameEquality(String descriptiveNameForDisplay, String origDesc, + byte[] startKey) { + // except for the "hidden-start-key" substring everything else should exactly match + String firstPart = descriptiveNameForDisplay.substring(0, + descriptiveNameForDisplay.indexOf(new String(HRegionInfo.HIDDEN_START_KEY))); + String secondPart = descriptiveNameForDisplay.substring( + descriptiveNameForDisplay.indexOf(new String(HRegionInfo.HIDDEN_START_KEY)) + + HRegionInfo.HIDDEN_START_KEY.length); + String firstPartOrig = origDesc.substring(0, + origDesc.indexOf(Bytes.toStringBinary(startKey))); + String secondPartOrig = origDesc.substring( + origDesc.indexOf(Bytes.toStringBinary(startKey)) + + Bytes.toStringBinary(startKey).length()); + assert(firstPart.equals(firstPartOrig)); + assert(secondPart.equals(secondPartOrig)); + } + private void checkEquality(HRegionInfo h, Configuration conf) throws IOException { + byte[] modifiedRegionName = HRegionInfo.getRegionNameForDisplay(h, conf); + byte[][] modifiedRegionNameParts = HRegionInfo.parseRegionName(modifiedRegionName); + byte[][] regionNameParts = HRegionInfo.parseRegionName(h.getRegionName()); + + //same number of parts + assert(modifiedRegionNameParts.length == regionNameParts.length); + + for (int i = 0; i < regionNameParts.length; i++) { + // all parts should match except for [1] where in the modified one, + // we should have "hidden_start_key" + if (i != 1) { + Assert.assertArrayEquals(regionNameParts[i], modifiedRegionNameParts[i]); + } else { + Assert.assertNotEquals(regionNameParts[i][0], modifiedRegionNameParts[i][0]); + Assert.assertArrayEquals(modifiedRegionNameParts[1], + HRegionInfo.getStartKeyForDisplay(h, conf)); + } + } + } }
