HBASE-17447 Implement a MasterObserver for automatically deleting space quotas
When a table or namespace is deleted, it would be nice to automatically delete the quota on said table/NS. It's possible that not all people would want this functionality so we can leave it up to the user to configure this Observer. Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/86709147 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/86709147 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/86709147 Branch: refs/heads/HBASE-16961 Commit: 8670914796800a3449c54729ccffca2334c52eaa Parents: 25096db Author: Josh Elser <els...@apache.org> Authored: Thu Mar 16 18:54:01 2017 -0400 Committer: Josh Elser <els...@apache.org> Committed: Mon May 22 13:08:12 2017 -0400 ---------------------------------------------------------------------- .../hbase/quotas/MasterSpaceQuotaObserver.java | 85 ++++++++++ .../quotas/TestMasterSpaceQuotaObserver.java | 169 +++++++++++++++++++ src/main/asciidoc/_chapters/ops_mgt.adoc | 17 ++ 3 files changed, 271 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/86709147/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java new file mode 100644 index 0000000..a3abf32 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.MasterObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; + +/** + * An observer to automatically delete space quotas when a table/namespace + * are deleted. + */ +@InterfaceAudience.Private +public class MasterSpaceQuotaObserver implements MasterObserver { + private CoprocessorEnvironment cpEnv; + private Configuration conf; + private boolean quotasEnabled = false; + + @Override + public void start(CoprocessorEnvironment ctx) throws IOException { + this.cpEnv = ctx; + this.conf = cpEnv.getConfiguration(); + this.quotasEnabled = QuotaUtil.isQuotaEnabled(conf); + } + + @Override + public void postDeleteTable( + ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException { + // Do nothing if quotas aren't enabled + if (!quotasEnabled) { + return; + } + final MasterServices master = ctx.getEnvironment().getMasterServices(); + final Connection conn = master.getConnection(); + Quotas quotas = QuotaUtil.getTableQuota(master.getConnection(), tableName); + if (null != quotas && quotas.hasSpace()) { + QuotaSettings settings = QuotaSettingsFactory.removeTableSpaceLimit(tableName); + try (Admin admin = conn.getAdmin()) { + admin.setQuota(settings); + } + } + } + + @Override + public void postDeleteNamespace( + ObserverContext<MasterCoprocessorEnvironment> ctx, String namespace) throws IOException { + // Do nothing if quotas aren't enabled + if (!quotasEnabled) { + return; + } + final MasterServices master = ctx.getEnvironment().getMasterServices(); + final Connection conn = master.getConnection(); + Quotas quotas = QuotaUtil.getNamespaceQuota(master.getConnection(), namespace); + if (null != quotas && quotas.hasSpace()) { + QuotaSettings settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(namespace); + try (Admin admin = conn.getAdmin()) { + admin.setQuota(settings); + } + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/86709147/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestMasterSpaceQuotaObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestMasterSpaceQuotaObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestMasterSpaceQuotaObserver.java new file mode 100644 index 0000000..a1eee4f --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestMasterSpaceQuotaObserver.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +/** + * Test class for {@link MasterSpaceQuotaObserver}. + */ +@Category(MediumTests.class) +public class TestMasterSpaceQuotaObserver { + private static final Log LOG = LogFactory.getLog(TestSpaceQuotas.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @Rule + public TestName testName = new TestName(); + + @BeforeClass + public static void setUp() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSpaceQuotaObserver.class.getName()); + conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void removeAllQuotas() throws Exception { + final Connection conn = TEST_UTIL.getConnection(); + // Wait for the quota table to be created + if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) { + do { + LOG.debug("Quota table does not yet exist"); + Thread.sleep(1000); + } while (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)); + } else { + // Or, clean up any quotas from previous test runs. + QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration()); + for (QuotaSettings quotaSettings : scanner) { + final String namespace = quotaSettings.getNamespace(); + final TableName tableName = quotaSettings.getTableName(); + if (null != namespace) { + LOG.debug("Deleting quota for namespace: " + namespace); + QuotaUtil.deleteNamespaceQuota(conn, namespace); + } else { + assert null != tableName; + LOG.debug("Deleting quota for table: "+ tableName); + QuotaUtil.deleteTableQuota(conn, tableName); + } + } + } + } + + @Test + public void testTableQuotaRemoved() throws Exception { + final Connection conn = TEST_UTIL.getConnection(); + final Admin admin = conn.getAdmin(); + final TableName tn = TableName.valueOf(testName.getMethodName()); + // Drop the table if it somehow exists + if (admin.tableExists(tn)) { + admin.disableTable(tn); + admin.deleteTable(tn); + } + + // Create a table + HTableDescriptor tableDesc = new HTableDescriptor(tn); + tableDesc.addFamily(new HColumnDescriptor("F1")); + admin.createTable(tableDesc); + assertEquals(0, getNumSpaceQuotas()); + + // Set a quota + QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( + tn, 1024L, SpaceViolationPolicy.NO_INSERTS); + admin.setQuota(settings); + assertEquals(1, getNumSpaceQuotas()); + + // Delete the table and observe the quota being automatically deleted as well + admin.disableTable(tn); + admin.deleteTable(tn); + assertEquals(0, getNumSpaceQuotas()); + } + + @Test + public void testNamespaceQuotaRemoved() throws Exception { + final Connection conn = TEST_UTIL.getConnection(); + final Admin admin = conn.getAdmin(); + final String ns = testName.getMethodName(); + // Drop the ns if it somehow exists + if (namespaceExists(ns)) { + admin.deleteNamespace(ns); + } + + // Create the ns + NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build(); + admin.createNamespace(desc); + assertEquals(0, getNumSpaceQuotas()); + + // Set a quota + QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace( + ns, 1024L, SpaceViolationPolicy.NO_INSERTS); + admin.setQuota(settings); + assertEquals(1, getNumSpaceQuotas()); + + // Delete the table and observe the quota being automatically deleted as well + admin.deleteNamespace(ns); + assertEquals(0, getNumSpaceQuotas()); + } + + public boolean namespaceExists(String ns) throws IOException { + NamespaceDescriptor[] descs = TEST_UTIL.getAdmin().listNamespaceDescriptors(); + for (NamespaceDescriptor desc : descs) { + if (ns.equals(desc.getName())) { + return true; + } + } + return false; + } + + public int getNumSpaceQuotas() throws Exception { + QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration()); + int numSpaceQuotas = 0; + for (QuotaSettings quotaSettings : scanner) { + if (quotaSettings.getQuotaType() == QuotaType.SPACE) { + numSpaceQuotas++; + } + } + return numSpaceQuotas; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/86709147/src/main/asciidoc/_chapters/ops_mgt.adoc ---------------------------------------------------------------------- diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc b/src/main/asciidoc/_chapters/ops_mgt.adoc index f9009f3..f60395b 100644 --- a/src/main/asciidoc/_chapters/ops_mgt.adoc +++ b/src/main/asciidoc/_chapters/ops_mgt.adoc @@ -1948,6 +1948,23 @@ and also will disallow all writes then the usage exceeds this limit. Because the on 'ns1:t1', this table can grow up to 100TB, but only if 'ns1:t2' and 'ns1:t3' have a usage of zero bytes. Practically, it's limit is 100TB less the current usage of 'ns1:t2' and 'ns1:t3'. +[[ops.space.quota.deletion]] +=== Automatic Space Quota Deletion + +By default, if a table or namespace is deleted that has a space quota, the quota itself is +not also deleted. In some cases, it may be desirable for the space quota to be automatically deleted. +In these cases, the user may configure the MasterSpaceQuotaObserver to delete any space quota +automatically in hbase-site.xml. + +[source,java] +---- + + <property> + <name>hbase.coprocessor.master.classes</name> + <value>...,org.apache.hadoop.hbase.quotas.MasterSpaceQuotaObserver</value> + </property> +---- + [[ops.backup]] == HBase Backup