Repository: hbase Updated Branches: refs/heads/master af466bf72 -> 5b485d14c
HBASE-17752 Shell command to list snapshot sizes WRT quotas Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5b485d14 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5b485d14 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5b485d14 Branch: refs/heads/master Commit: 5b485d14cd6a5bce1bef8329122166153162037e Parents: af466bf Author: Josh Elser <[email protected]> Authored: Fri Mar 24 14:00:13 2017 -0400 Committer: Josh Elser <[email protected]> Committed: Tue Jun 20 14:17:00 2017 -0400 ---------------------------------------------------------------------- .../hadoop/hbase/quotas/QuotaTableUtil.java | 61 ++++++++++++++++---- .../hbase/quotas/TableQuotaSnapshotStore.java | 2 +- .../quotas/TestSpaceQuotasWithSnapshots.java | 14 +++++ hbase-shell/src/main/ruby/hbase/quotas.rb | 4 ++ hbase-shell/src/main/ruby/shell.rb | 1 + .../ruby/shell/commands/list_snapshot_sizes.rb | 42 ++++++++++++++ hbase-shell/src/test/ruby/hbase/quotas_test.rb | 29 +++++++++- hbase-shell/src/test/ruby/tests_runner.rb | 2 + 8 files changed, 143 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/5b485d14/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java index c1863a7..d1bbade 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java @@ -31,6 +31,7 @@ import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -513,16 +514,50 @@ public class QuotaTableUtil { return QuotaProtos.SpaceQuotaSnapshot.parseFrom(bs).getQuotaUsage(); } - static Scan createScanForSnapshotSizes(TableName table) { - byte[] rowkey = getTableRowKey(table); - return new Scan() - // Fetch just this one row - .withStartRow(rowkey) - .withStopRow(rowkey, true) - // Just the usage family - .addFamily(QUOTA_FAMILY_USAGE) - // Only the snapshot size qualifiers - .setFilter(new ColumnPrefixFilter(QUOTA_SNAPSHOT_SIZE_QUALIFIER)); + static Scan createScanForSpaceSnapshotSizes() { + return createScanForSpaceSnapshotSizes(null); + } + + static Scan createScanForSpaceSnapshotSizes(TableName table) { + Scan s = new Scan(); + if (null == table) { + // Read all tables, just look at the row prefix + s.setRowPrefixFilter(QUOTA_TABLE_ROW_KEY_PREFIX); + } else { + // Fetch the exact row for the table + byte[] rowkey = getTableRowKey(table); + // Fetch just this one row + s.withStartRow(rowkey).withStopRow(rowkey, true); + } + + // Just the usage family and only the snapshot size qualifiers + return s.addFamily(QUOTA_FAMILY_USAGE).setFilter( + new ColumnPrefixFilter(QUOTA_SNAPSHOT_SIZE_QUALIFIER)); + } + + /** + * Fetches any persisted HBase snapshot sizes stored in the quota table. The sizes here are + * computed relative to the table which the snapshot was created from. A snapshot's size will + * not include the size of files which the table still refers. These sizes, in bytes, are what + * is used internally to compute quota violation for tables and namespaces. + * + * @return A map of snapshot name to size in bytes per space quota computations + */ + public static Map<String,Long> getObservedSnapshotSizes(Connection conn) throws IOException { + try (Table quotaTable = conn.getTable(QUOTA_TABLE_NAME); + ResultScanner rs = quotaTable.getScanner(createScanForSpaceSnapshotSizes())) { + final Map<String,Long> snapshotSizes = new HashMap<>(); + for (Result r : rs) { + CellScanner cs = r.cellScanner(); + while (cs.advance()) { + Cell c = cs.current(); + final String snapshot = extractSnapshotNameFromSizeCell(c); + final long size = parseSnapshotSize(c); + snapshotSizes.put(snapshot, size); + } + } + return snapshotSizes; + } } /* ========================================================================= @@ -749,6 +784,12 @@ public class QuotaTableUtil { return Bytes.add(QUOTA_SNAPSHOT_SIZE_QUALIFIER, Bytes.toBytes(snapshotName)); } + protected static String extractSnapshotNameFromSizeCell(Cell c) { + return Bytes.toString( + c.getQualifierArray(), c.getQualifierOffset() + QUOTA_SNAPSHOT_SIZE_QUALIFIER.length, + c.getQualifierLength() - QUOTA_SNAPSHOT_SIZE_QUALIFIER.length); + } + protected static long extractSnapshotSize( byte[] data, int offset, int length) throws InvalidProtocolBufferException { ByteString byteStr = UnsafeByteOperations.unsafeWrap(data, offset, length); http://git-wip-us.apache.org/repos/asf/hbase/blob/5b485d14/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java index 6a29a82..b94e643 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java @@ -114,7 +114,7 @@ public class TableQuotaSnapshotStore implements QuotaSnapshotStore<TableName> { */ long getSnapshotSizesForTable(TableName tn) throws IOException { try (Table quotaTable = conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) { - Scan s = QuotaTableUtil.createScanForSnapshotSizes(tn); + Scan s = QuotaTableUtil.createScanForSpaceSnapshotSizes(tn); ResultScanner rs = quotaTable.getScanner(s); try { long size = 0L; http://git-wip-us.apache.org/repos/asf/hbase/blob/5b485d14/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java index ebb1a9e..85c7de2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java @@ -173,6 +173,13 @@ public class TestSpaceQuotasWithSnapshots { return expectedFinalSize == snapshot.getUsage(); } }); + + Map<String,Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn); + Long size = snapshotSizes.get(snapshot1); + assertNotNull("Did not observe the size of the snapshot", size); + assertEquals( + "The recorded size of the HBase snapshot was not the size we expected", actualInitialSize, + size.longValue()); } @Test @@ -265,6 +272,13 @@ public class TestSpaceQuotasWithSnapshots { return expectedFinalSize == snapshot.getUsage(); } }); + + Map<String,Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn); + Long size = snapshotSizes.get(snapshot1); + assertNotNull("Did not observe the size of the snapshot", size); + assertEquals( + "The recorded size of the HBase snapshot was not the size we expected", actualInitialSize, + size.longValue()); } @Test http://git-wip-us.apache.org/repos/asf/hbase/blob/5b485d14/hbase-shell/src/main/ruby/hbase/quotas.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb index 784896e..a8a8e6b 100644 --- a/hbase-shell/src/main/ruby/hbase/quotas.rb +++ b/hbase-shell/src/main/ruby/hbase/quotas.rb @@ -241,6 +241,10 @@ module Hbase return count end + def list_snapshot_sizes() + QuotaTableUtil.getObservedSnapshotSizes(@admin.getConnection()) + end + def _parse_size(str_limit) str_limit = str_limit.downcase match = /(\d+)([bkmgtp%]*)/.match(str_limit) http://git-wip-us.apache.org/repos/asf/hbase/blob/5b485d14/hbase-shell/src/main/ruby/shell.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index aaf26b3..ab4d514 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -423,6 +423,7 @@ Shell.load_command_group( list_quotas list_quota_table_sizes list_quota_snapshots + list_snapshot_sizes ] ) http://git-wip-us.apache.org/repos/asf/hbase/blob/5b485d14/hbase-shell/src/main/ruby/shell/commands/list_snapshot_sizes.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/list_snapshot_sizes.rb b/hbase-shell/src/main/ruby/shell/commands/list_snapshot_sizes.rb new file mode 100644 index 0000000..4ab8db9 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/list_snapshot_sizes.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListSnapshotSizes < Command + def help + return <<-EOF +Lists the size of every HBase snapshot given the space quota size computation +algorithms. An HBase snapshot only "owns" the size of a file when the table +from which the snapshot was created no longer refers to that file. +EOF + end + + def command(args = {}) + formatter.header(["SNAPSHOT", "SIZE"]) + count = 0 + quotas_admin.list_snapshot_sizes().each do |snapshot,size| + formatter.row([snapshot.to_s, size.to_s]) + count += 1 + end + formatter.footer(count) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/5b485d14/hbase-shell/src/test/ruby/hbase/quotas_test.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/test/ruby/hbase/quotas_test.rb b/hbase-shell/src/test/ruby/hbase/quotas_test.rb index 076eaed..3fb00c8 100644 --- a/hbase-shell/src/test/ruby/hbase/quotas_test.rb +++ b/hbase-shell/src/test/ruby/hbase/quotas_test.rb @@ -32,7 +32,7 @@ module Hbase def setup setup_hbase # Create test table if it does not exist - @test_name = "hbase_shell_tests_table" + @test_name = "hbase_shell_quota_tests_table" create_test_table(@test_name) end @@ -109,5 +109,32 @@ module Hbase output = capture_stdout{ command(:list_quotas) } assert(output.include?("0 row(s)")) end + + define_test 'can view size of snapshots' do + snapshot1 = "#{@test_name}_1" + snapshot2 = "#{@test_name}_2" + # Set a quota on our table + command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS, TABLE => @test_name) + (1..10).each{|i| command(:put, @test_name, 'a', "x:#{i}", "#{i}")} + command(:flush, @test_name) + command(:snapshot, @test_name, snapshot1) + (1..10).each{|i| command(:put, @test_name, 'b', "x:#{i}", "#{i}")} + command(:flush, @test_name) + command(:snapshot, @test_name, snapshot2) + duration_to_check = 1000 * 30 + start = current = Time.now.to_i + # Poor man's Waiter from Java test classes + while current - start < duration_to_check + output = capture_stdout{ command(:list_snapshot_sizes) } + if output.include? snapshot1 and output.include? snapshot2 + break + end + sleep 5 + current = Time.now.to_i + end + output = capture_stdout{ command(:list_snapshot_sizes) } + assert(output.include? snapshot1) + assert(output.include? snapshot2) + end end end http://git-wip-us.apache.org/repos/asf/hbase/blob/5b485d14/hbase-shell/src/test/ruby/tests_runner.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/test/ruby/tests_runner.rb b/hbase-shell/src/test/ruby/tests_runner.rb index 54bf3f9..73d4a6e 100644 --- a/hbase-shell/src/test/ruby/tests_runner.rb +++ b/hbase-shell/src/test/ruby/tests_runner.rb @@ -37,6 +37,8 @@ unless defined?($TEST_CLUSTER) $TEST_CLUSTER.configuration.setInt("hbase.regionserver.msginterval", 100) $TEST_CLUSTER.configuration.setInt("hbase.client.pause", 250) $TEST_CLUSTER.configuration.set("hbase.quota.enabled", "true") + $TEST_CLUSTER.configuration.set('hbase.master.quotas.snapshot.chore.period', 5000) + $TEST_CLUSTER.configuration.set('hbase.master.quotas.snapshot.chore.delay', 5000) $TEST_CLUSTER.configuration.setInt(org.apache.hadoop.hbase.HConstants::HBASE_CLIENT_RETRIES_NUMBER, 6) $TEST_CLUSTER.startMiniCluster @own_cluster = true
