Repository: hbase Updated Branches: refs/heads/branch-1 bbdd50b9c -> c031d8de2
http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java new file mode 100644 index 0000000..8d33cd2 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ + +package org.apache.hadoop.hbase.quotas; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Throttle; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ SmallTests.class }) +public class TestQuotaState { + private static final TableName UNKNOWN_TABLE_NAME = TableName.valueOf("unknownTable"); + + @Test(timeout = 60000) + public void testQuotaStateBypass() { + QuotaState quotaInfo = new QuotaState(); + assertTrue(quotaInfo.isBypass()); + assertNoopLimiter(quotaInfo.getGlobalLimiter()); + + UserQuotaState userQuotaState = new UserQuotaState(); + assertTrue(userQuotaState.isBypass()); + assertNoopLimiter(userQuotaState.getTableLimiter(UNKNOWN_TABLE_NAME)); + } + + @Test(timeout = 60000) + public void testSimpleQuotaStateOperation() { + final TableName table = TableName.valueOf("testSimpleQuotaStateOperationTable"); + final int NUM_GLOBAL_THROTTLE = 3; + final int NUM_TABLE_THROTTLE = 2; + + UserQuotaState quotaInfo = new UserQuotaState(); + assertTrue(quotaInfo.isBypass()); + + // Set global quota + quotaInfo.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE)); + assertFalse(quotaInfo.isBypass()); + + // Set table quota + quotaInfo.setQuotas(table, buildReqNumThrottle(NUM_TABLE_THROTTLE)); + assertFalse(quotaInfo.isBypass()); + assertTrue(quotaInfo.getGlobalLimiter() == quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); + assertThrottleException(quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME), NUM_GLOBAL_THROTTLE); + assertThrottleException(quotaInfo.getTableLimiter(table), NUM_TABLE_THROTTLE); + } + + @Test(timeout = 60000) + public void testQuotaStateUpdateBypassThrottle() { + final long LAST_UPDATE = 10; + + UserQuotaState quotaInfo = new UserQuotaState(); + assertEquals(0, quotaInfo.getLastUpdate()); + assertTrue(quotaInfo.isBypass()); + + UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE); + assertEquals(LAST_UPDATE, otherQuotaState.getLastUpdate()); + assertTrue(otherQuotaState.isBypass()); + + quotaInfo.update(otherQuotaState); + assertEquals(LAST_UPDATE, quotaInfo.getLastUpdate()); + assertTrue(quotaInfo.isBypass()); + assertTrue(quotaInfo.getGlobalLimiter() == quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); + assertNoopLimiter(quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); + } + + @Test(timeout = 60000) + public void testQuotaStateUpdateGlobalThrottle() { + final int NUM_GLOBAL_THROTTLE_1 = 3; + final int NUM_GLOBAL_THROTTLE_2 = 11; + final long LAST_UPDATE_1 = 10; + final long LAST_UPDATE_2 = 20; + final long LAST_UPDATE_3 = 30; + + QuotaState quotaInfo = new QuotaState(); + assertEquals(0, quotaInfo.getLastUpdate()); + assertTrue(quotaInfo.isBypass()); + + // Add global throttle + QuotaState otherQuotaState = new QuotaState(LAST_UPDATE_1); + otherQuotaState.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE_1)); + assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); + assertFalse(otherQuotaState.isBypass()); + + quotaInfo.update(otherQuotaState); + assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); + assertFalse(quotaInfo.isBypass()); + assertThrottleException(quotaInfo.getGlobalLimiter(), NUM_GLOBAL_THROTTLE_1); + + // Update global Throttle + otherQuotaState = new QuotaState(LAST_UPDATE_2); + otherQuotaState.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE_2)); + assertEquals(LAST_UPDATE_2, otherQuotaState.getLastUpdate()); + assertFalse(otherQuotaState.isBypass()); + + quotaInfo.update(otherQuotaState); + assertEquals(LAST_UPDATE_2, quotaInfo.getLastUpdate()); + assertFalse(quotaInfo.isBypass()); + assertThrottleException(quotaInfo.getGlobalLimiter(), NUM_GLOBAL_THROTTLE_2 + - NUM_GLOBAL_THROTTLE_1); + + // Remove global throttle + otherQuotaState = new QuotaState(LAST_UPDATE_3); + assertEquals(LAST_UPDATE_3, otherQuotaState.getLastUpdate()); + assertTrue(otherQuotaState.isBypass()); + + quotaInfo.update(otherQuotaState); + assertEquals(LAST_UPDATE_3, quotaInfo.getLastUpdate()); + assertTrue(quotaInfo.isBypass()); + assertNoopLimiter(quotaInfo.getGlobalLimiter()); + } + + @Test(timeout = 60000) + public void testQuotaStateUpdateTableThrottle() { + final TableName TABLE_A = TableName.valueOf("TableA"); + final TableName TABLE_B = TableName.valueOf("TableB"); + final TableName TABLE_C = TableName.valueOf("TableC"); + final int TABLE_A_THROTTLE_1 = 3; + final int TABLE_A_THROTTLE_2 = 11; + final int TABLE_B_THROTTLE = 4; + final int TABLE_C_THROTTLE = 5; + final long LAST_UPDATE_1 = 10; + final long LAST_UPDATE_2 = 20; + final long LAST_UPDATE_3 = 30; + + UserQuotaState quotaInfo = new UserQuotaState(); + assertEquals(0, quotaInfo.getLastUpdate()); + assertTrue(quotaInfo.isBypass()); + + // Add A B table limiters + UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE_1); + otherQuotaState.setQuotas(TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_1)); + otherQuotaState.setQuotas(TABLE_B, buildReqNumThrottle(TABLE_B_THROTTLE)); + assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); + assertFalse(otherQuotaState.isBypass()); + + quotaInfo.update(otherQuotaState); + assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); + assertFalse(quotaInfo.isBypass()); + assertThrottleException(quotaInfo.getTableLimiter(TABLE_A), TABLE_A_THROTTLE_1); + assertThrottleException(quotaInfo.getTableLimiter(TABLE_B), TABLE_B_THROTTLE); + assertNoopLimiter(quotaInfo.getTableLimiter(TABLE_C)); + + // Add C, Remove B, Update A table limiters + otherQuotaState = new UserQuotaState(LAST_UPDATE_2); + otherQuotaState.setQuotas(TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_2)); + otherQuotaState.setQuotas(TABLE_C, buildReqNumThrottle(TABLE_C_THROTTLE)); + assertEquals(LAST_UPDATE_2, otherQuotaState.getLastUpdate()); + assertFalse(otherQuotaState.isBypass()); + + quotaInfo.update(otherQuotaState); + assertEquals(LAST_UPDATE_2, quotaInfo.getLastUpdate()); + assertFalse(quotaInfo.isBypass()); + assertThrottleException(quotaInfo.getTableLimiter(TABLE_A), TABLE_A_THROTTLE_2 + - TABLE_A_THROTTLE_1); + assertThrottleException(quotaInfo.getTableLimiter(TABLE_C), TABLE_C_THROTTLE); + assertNoopLimiter(quotaInfo.getTableLimiter(TABLE_B)); + + // Remove table limiters + otherQuotaState = new UserQuotaState(LAST_UPDATE_3); + assertEquals(LAST_UPDATE_3, otherQuotaState.getLastUpdate()); + assertTrue(otherQuotaState.isBypass()); + + quotaInfo.update(otherQuotaState); + assertEquals(LAST_UPDATE_3, quotaInfo.getLastUpdate()); + assertTrue(quotaInfo.isBypass()); + assertNoopLimiter(quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); + } + + private Quotas buildReqNumThrottle(final long limit) { + return Quotas + .newBuilder() + .setThrottle( + Throttle.newBuilder() + .setReqNum(ProtobufUtil.toTimedQuota(limit, TimeUnit.MINUTES, QuotaScope.MACHINE)) + .build()).build(); + } + + private void assertThrottleException(final QuotaLimiter limiter, final int availReqs) { + assertNoThrottleException(limiter, availReqs); + try { + limiter.checkQuota(1, 1); + fail("Should have thrown ThrottlingException"); + } catch (ThrottlingException e) { + // expected + } + } + + private void assertNoThrottleException(final QuotaLimiter limiter, final int availReqs) { + for (int i = 0; i < availReqs; ++i) { + try { + limiter.checkQuota(1, 1); + } catch (ThrottlingException e) { + fail("Unexpected ThrottlingException after " + i + " requests. limit=" + availReqs); + } + limiter.grabQuota(1, 1); + } + } + + private void assertNoopLimiter(final QuotaLimiter limiter) { + assertTrue(limiter == NoopQuotaLimiter.get()); + assertNoThrottleException(limiter, 100); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java new file mode 100644 index 0000000..fee5a10 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ + +package org.apache.hadoop.hbase.quotas; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Throttle; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test the quota table helpers (e.g. CRUD operations) + */ +@Category({ MediumTests.class }) +public class TestQuotaTableUtil { + final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private Connection connection; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 2000); + TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); + TEST_UTIL.startMiniCluster(1); + TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void before() throws IOException { + this.connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); + } + + @After + public void after() throws IOException { + this.connection.close(); + } + + @Test + public void testTableQuotaUtil() throws Exception { + final TableName table = TableName.valueOf("testTableQuotaUtilTable"); + + Quotas quota = + Quotas + .newBuilder() + .setThrottle( + Throttle + .newBuilder() + .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE)) + .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE)) + .setReadSize( + ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE)).build()) + .build(); + + // Add user quota and verify it + QuotaUtil.addTableQuota(this.connection, table, quota); + Quotas resQuota = QuotaUtil.getTableQuota(this.connection, table); + assertEquals(quota, resQuota); + + // Remove user quota and verify it + QuotaUtil.deleteTableQuota(this.connection, table); + resQuota = QuotaUtil.getTableQuota(this.connection, table); + assertEquals(null, resQuota); + } + + @Test + public void testNamespaceQuotaUtil() throws Exception { + final String namespace = "testNamespaceQuotaUtilNS"; + + Quotas quota = + Quotas + .newBuilder() + .setThrottle( + Throttle + .newBuilder() + .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE)) + .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE)) + .setReadSize( + ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE)).build()) + .build(); + + // Add user quota and verify it + QuotaUtil.addNamespaceQuota(this.connection, namespace, quota); + Quotas resQuota = QuotaUtil.getNamespaceQuota(this.connection, namespace); + assertEquals(quota, resQuota); + + // Remove user quota and verify it + QuotaUtil.deleteNamespaceQuota(this.connection, namespace); + resQuota = QuotaUtil.getNamespaceQuota(this.connection, namespace); + assertEquals(null, resQuota); + } + + @Test + public void testUserQuotaUtil() throws Exception { + final TableName table = TableName.valueOf("testUserQuotaUtilTable"); + final String namespace = "testNS"; + final String user = "testUser"; + + Quotas quotaNamespace = + Quotas + .newBuilder() + .setThrottle( + Throttle + .newBuilder() + .setReqNum(ProtobufUtil.toTimedQuota(50000, TimeUnit.SECONDS, QuotaScope.MACHINE)) + .build()).build(); + Quotas quotaTable = + Quotas + .newBuilder() + .setThrottle( + Throttle + .newBuilder() + .setReqNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE)) + .setWriteNum(ProtobufUtil.toTimedQuota(600, TimeUnit.SECONDS, QuotaScope.MACHINE)) + .setReadSize( + ProtobufUtil.toTimedQuota(10000, TimeUnit.SECONDS, QuotaScope.MACHINE)).build()) + .build(); + Quotas quota = + Quotas + .newBuilder() + .setThrottle( + Throttle + .newBuilder() + .setReqSize(ProtobufUtil.toTimedQuota(8192, TimeUnit.SECONDS, QuotaScope.MACHINE)) + .setWriteSize( + ProtobufUtil.toTimedQuota(4096, TimeUnit.SECONDS, QuotaScope.MACHINE)) + .setReadNum(ProtobufUtil.toTimedQuota(1000, TimeUnit.SECONDS, QuotaScope.MACHINE)) + .build()).build(); + + // Add user global quota + QuotaUtil.addUserQuota(this.connection, user, quota); + Quotas resQuota = QuotaUtil.getUserQuota(this.connection, user); + assertEquals(quota, resQuota); + + // Add user quota for table + QuotaUtil.addUserQuota(this.connection, user, table, quotaTable); + Quotas resQuotaTable = QuotaUtil.getUserQuota(this.connection, user, table); + assertEquals(quotaTable, resQuotaTable); + + // Add user quota for namespace + QuotaUtil.addUserQuota(this.connection, user, namespace, quotaNamespace); + Quotas resQuotaNS = QuotaUtil.getUserQuota(this.connection, user, namespace); + assertEquals(quotaNamespace, resQuotaNS); + + // Delete user global quota + QuotaUtil.deleteUserQuota(this.connection, user); + resQuota = QuotaUtil.getUserQuota(this.connection, user); + assertEquals(null, resQuota); + + // Delete user quota for table + QuotaUtil.deleteUserQuota(this.connection, user, table); + resQuotaTable = QuotaUtil.getUserQuota(this.connection, user, table); + assertEquals(null, resQuotaTable); + + // Delete user quota for namespace + QuotaUtil.deleteUserQuota(this.connection, user, namespace); + resQuotaNS = QuotaUtil.getUserQuota(this.connection, user, namespace); + assertEquals(null, resQuotaNS); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java new file mode 100644 index 0000000..4bb0d35 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java @@ -0,0 +1,409 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ + +package org.apache.hadoop.hbase.quotas; + +import static org.junit.Assert.assertEquals; + +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; +import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MediumTests.class }) +public class TestQuotaThrottle { + final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private final static byte[] FAMILY = Bytes.toBytes("cf"); + private final static byte[] QUALIFIER = Bytes.toBytes("q"); + + private final static TableName[] TABLE_NAMES = new TableName[] { + TableName.valueOf("TestQuotaAdmin0"), TableName.valueOf("TestQuotaAdmin1"), + TableName.valueOf("TestQuotaAdmin2") }; + + private static HTable[] tables; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); + TEST_UTIL.startMiniCluster(1); + TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); + QuotaCache.setTEST_FORCE_REFRESH(true); + + tables = new HTable[TABLE_NAMES.length]; + for (int i = 0; i < TABLE_NAMES.length; ++i) { + tables[i] = TEST_UTIL.createTable(TABLE_NAMES[i], FAMILY); + } + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + for (int i = 0; i < tables.length; ++i) { + if (tables[i] != null) { + tables[i].close(); + TEST_UTIL.deleteTable(TABLE_NAMES[i]); + } + } + + TEST_UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) { + RegionServerQuotaManager quotaManager = rst.getRegionServer().getRegionServerQuotaManager(); + QuotaCache quotaCache = quotaManager.getQuotaCache(); + quotaCache.getNamespaceQuotaCache().clear(); + quotaCache.getTableQuotaCache().clear(); + quotaCache.getUserQuotaCache().clear(); + } + } + + @Test(timeout = 60000) + public void testUserGlobalThrottle() throws Exception { + final Admin admin = TEST_UTIL.getHBaseAdmin(); + final String userName = User.getCurrent().getShortName(); + + // Add 6req/min limit + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 6, + TimeUnit.MINUTES)); + triggerUserCacheRefresh(false, TABLE_NAMES); + + // should execute at max 6 requests + assertEquals(6, doPuts(100, tables)); + + // wait a minute and you should get other 6 requests executed + waitMinuteQuota(); + assertEquals(6, doPuts(100, tables)); + + // Remove all the limits + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName)); + triggerUserCacheRefresh(true, TABLE_NAMES); + assertEquals(60, doPuts(60, tables)); + assertEquals(60, doGets(60, tables)); + } + + @Test(timeout = 60000) + public void testUserTableThrottle() throws Exception { + final Admin admin = TEST_UTIL.getHBaseAdmin(); + final String userName = User.getCurrent().getShortName(); + + // Add 6req/min limit + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[0], + ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES)); + triggerUserCacheRefresh(false, TABLE_NAMES[0]); + + // should execute at max 6 requests on tables[0] and have no limit on tables[1] + assertEquals(6, doPuts(100, tables[0])); + assertEquals(30, doPuts(30, tables[1])); + + // wait a minute and you should get other 6 requests executed + waitMinuteQuota(); + assertEquals(6, doPuts(100, tables[0])); + + // Remove all the limits + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0])); + triggerUserCacheRefresh(true, TABLE_NAMES); + assertEquals(60, doPuts(60, tables)); + assertEquals(60, doGets(60, tables)); + } + + @Test(timeout = 60000) + public void testUserNamespaceThrottle() throws Exception { + final Admin admin = TEST_UTIL.getHBaseAdmin(); + final String userName = User.getCurrent().getShortName(); + final String NAMESPACE = "default"; + + // Add 6req/min limit + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, NAMESPACE, + ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES)); + triggerUserCacheRefresh(false, TABLE_NAMES[0]); + + // should execute at max 6 requests on tables[0] and have no limit on tables[1] + assertEquals(6, doPuts(100, tables[0])); + + // wait a minute and you should get other 6 requests executed + waitMinuteQuota(); + assertEquals(6, doPuts(100, tables[1])); + + // Remove all the limits + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, NAMESPACE)); + triggerUserCacheRefresh(true, TABLE_NAMES); + assertEquals(60, doPuts(60, tables)); + assertEquals(60, doGets(60, tables)); + } + + @Test(timeout = 60000) + public void testTableGlobalThrottle() throws Exception { + final Admin admin = TEST_UTIL.getHBaseAdmin(); + + // Add 6req/min limit + admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER, + 6, TimeUnit.MINUTES)); + triggerTableCacheRefresh(false, TABLE_NAMES[0]); + + // should execute at max 6 requests + assertEquals(6, doPuts(100, tables[0])); + // should have no limits + assertEquals(30, doPuts(30, tables[1])); + + // wait a minute and you should get other 6 requests executed + waitMinuteQuota(); + assertEquals(6, doPuts(100, tables[0])); + + // Remove all the limits + admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0])); + triggerTableCacheRefresh(true, TABLE_NAMES[0]); + assertEquals(80, doGets(80, tables[0], tables[1])); + } + + @Test(timeout = 60000) + public void testNamespaceGlobalThrottle() throws Exception { + final Admin admin = TEST_UTIL.getHBaseAdmin(); + final String NAMESPACE = "default"; + + // Add 6req/min limit + admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.REQUEST_NUMBER, + 6, TimeUnit.MINUTES)); + triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]); + + // should execute at max 6 requests + assertEquals(6, doPuts(100, tables[0])); + + // wait a minute and you should get other 6 requests executed + waitMinuteQuota(); + assertEquals(6, doPuts(100, tables[1])); + + admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE)); + triggerNamespaceCacheRefresh(true, TABLE_NAMES[0]); + assertEquals(40, doPuts(40, tables[0])); + } + + @Test(timeout = 60000) + public void testUserAndTableThrottle() throws Exception { + final Admin admin = TEST_UTIL.getHBaseAdmin(); + final String userName = User.getCurrent().getShortName(); + + // Add 6req/min limit for the user on tables[0] + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[0], + ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES)); + triggerUserCacheRefresh(false, TABLE_NAMES[0]); + // Add 12req/min limit for the user + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 12, + TimeUnit.MINUTES)); + triggerUserCacheRefresh(false, TABLE_NAMES[1], TABLE_NAMES[2]); + // Add 8req/min limit for the tables[1] + admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[1], ThrottleType.REQUEST_NUMBER, + 8, TimeUnit.MINUTES)); + triggerTableCacheRefresh(false, TABLE_NAMES[1]); + // Add a lower table level throttle on tables[0] + admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER, + 3, TimeUnit.MINUTES)); + triggerTableCacheRefresh(false, TABLE_NAMES[0]); + + // should execute at max 12 requests + assertEquals(12, doGets(100, tables[2])); + + // should execute at max 8 requests + waitMinuteQuota(); + assertEquals(8, doGets(100, tables[1])); + + // should execute at max 3 requests + waitMinuteQuota(); + assertEquals(3, doPuts(100, tables[0])); + + // Remove all the throttling rules + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0])); + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName)); + triggerUserCacheRefresh(true, TABLE_NAMES[0], TABLE_NAMES[1]); + + admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[1])); + triggerTableCacheRefresh(true, TABLE_NAMES[1]); + waitMinuteQuota(); + assertEquals(40, doGets(40, tables[1])); + + admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0])); + triggerTableCacheRefresh(true, TABLE_NAMES[0]); + waitMinuteQuota(); + assertEquals(40, doGets(40, tables[0])); + } + + @Test(timeout = 60000) + public void testUserGlobalBypassThrottle() throws Exception { + final Admin admin = TEST_UTIL.getHBaseAdmin(); + final String userName = User.getCurrent().getShortName(); + final String NAMESPACE = "default"; + + // Add 6req/min limit for tables[0] + admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER, + 6, TimeUnit.MINUTES)); + triggerTableCacheRefresh(false, TABLE_NAMES[0]); + // Add 13req/min limit for the user + admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.REQUEST_NUMBER, + 13, TimeUnit.MINUTES)); + triggerNamespaceCacheRefresh(false, TABLE_NAMES[1]); + + // should execute at max 6 requests on table[0] and (13 - 6) on table[1] + assertEquals(6, doPuts(100, tables[0])); + assertEquals(7, doGets(100, tables[1])); + waitMinuteQuota(); + + // Set the global bypass for the user + admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true)); + admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[2], + ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES)); + triggerUserCacheRefresh(false, TABLE_NAMES[2]); + assertEquals(30, doGets(30, tables[0])); + assertEquals(30, doGets(30, tables[1])); + waitMinuteQuota(); + + // Remove the global bypass + // should execute at max 6 requests on table[0] and (13 - 6) on table[1] + admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false)); + admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[2])); + triggerUserCacheRefresh(true, TABLE_NAMES[2]); + assertEquals(6, doPuts(100, tables[0])); + assertEquals(7, doGets(100, tables[1])); + + // unset throttle + admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0])); + admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE)); + waitMinuteQuota(); + triggerTableCacheRefresh(true, TABLE_NAMES[0]); + triggerNamespaceCacheRefresh(true, TABLE_NAMES[1]); + assertEquals(30, doGets(30, tables[0])); + assertEquals(30, doGets(30, tables[1])); + } + + private int doPuts(int maxOps, final HTable... tables) throws Exception { + int count = 0; + try { + while (count < maxOps) { + Put put = new Put(Bytes.toBytes("row-" + count)); + put.addColumn(FAMILY, QUALIFIER, Bytes.toBytes("data-" + count)); + for (final HTable table : tables) { + table.put(put); + } + count += tables.length; + } + } catch (RetriesExhaustedWithDetailsException e) { + for (Throwable t : e.getCauses()) { + if (!(t instanceof ThrottlingException)) { + throw e; + } + } + LOG.error("put failed after nRetries=" + count, e); + } + return count; + } + + private long doGets(int maxOps, final HTable... tables) throws Exception { + int count = 0; + try { + while (count < maxOps) { + Get get = new Get(Bytes.toBytes("row-" + count)); + for (final HTable table : tables) { + table.get(get); + } + count += tables.length; + } + } catch (ThrottlingException e) { + LOG.error("get failed after nRetries=" + count, e); + } + return count; + } + + private void triggerUserCacheRefresh(boolean bypass, TableName... tables) throws Exception { + triggerCacheRefresh(bypass, true, false, false, tables); + } + + private void triggerTableCacheRefresh(boolean bypass, TableName... tables) throws Exception { + triggerCacheRefresh(bypass, false, true, false, tables); + } + + private void triggerNamespaceCacheRefresh(boolean bypass, TableName... tables) throws Exception { + triggerCacheRefresh(bypass, false, false, true, tables); + } + + private void triggerCacheRefresh(boolean bypass, boolean userLimiter, boolean tableLimiter, + boolean nsLimiter, final TableName... tables) throws Exception { + for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) { + RegionServerQuotaManager quotaManager = rst.getRegionServer().getRegionServerQuotaManager(); + QuotaCache quotaCache = quotaManager.getQuotaCache(); + + quotaCache.triggerCacheRefresh(); + Thread.sleep(250); + + for (TableName table : tables) { + quotaCache.getTableLimiter(table); + } + + boolean isUpdated = false; + while (!isUpdated) { + isUpdated = true; + for (TableName table : tables) { + boolean isBypass = true; + if (userLimiter) { + isBypass &= quotaCache.getUserLimiter(User.getCurrent().getUGI(), table).isBypass(); + } + if (tableLimiter) { + isBypass &= quotaCache.getTableLimiter(table).isBypass(); + } + if (nsLimiter) { + isBypass &= quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass(); + } + if (isBypass != bypass) { + isUpdated = false; + Thread.sleep(250); + break; + } + } + } + + LOG.debug("QuotaCache"); + LOG.debug(quotaCache.getNamespaceQuotaCache()); + LOG.debug(quotaCache.getTableQuotaCache()); + LOG.debug(quotaCache.getUserQuotaCache()); + } + } + + private void waitMinuteQuota() { + EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge( + EnvironmentEdgeManager.currentTime() + 70000)); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestRateLimiter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestRateLimiter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestRateLimiter.java new file mode 100644 index 0000000..765f321 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestRateLimiter.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ + +package org.apache.hadoop.hbase.quotas; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Verify the behaviour of the Rate Limiter. + */ +@Category({ SmallTests.class }) +public class TestRateLimiter { + @Test + public void testWaitIntervalTimeUnitSeconds() { + testWaitInterval(TimeUnit.SECONDS, 10, 100); + } + + @Test + public void testWaitIntervalTimeUnitMinutes() { + testWaitInterval(TimeUnit.MINUTES, 10, 6000); + } + + @Test + public void testWaitIntervalTimeUnitHours() { + testWaitInterval(TimeUnit.HOURS, 10, 360000); + } + + @Test + public void testWaitIntervalTimeUnitDays() { + testWaitInterval(TimeUnit.DAYS, 10, 8640000); + } + + private void testWaitInterval(final TimeUnit timeUnit, final long limit, + final long expectedWaitInterval) { + RateLimiter limiter = new RateLimiter(); + limiter.set(limit, timeUnit); + + long nowTs = 0; + long lastTs = 0; + + // consume all the available resources, one request at the time. + // the wait interval should be 0 + for (int i = 0; i < (limit - 1); ++i) { + assertTrue(limiter.canExecute(nowTs, lastTs)); + limiter.consume(); + long waitInterval = limiter.waitInterval(); + assertEquals(0, waitInterval); + } + + for (int i = 0; i < (limit * 4); ++i) { + // There is one resource available, so we should be able to + // consume it without waiting. + assertTrue(limiter.canExecute(nowTs, lastTs)); + assertEquals(0, limiter.waitInterval()); + limiter.consume(); + lastTs = nowTs; + + // No more resources are available, we should wait for at least an interval. + long waitInterval = limiter.waitInterval(); + assertEquals(expectedWaitInterval, waitInterval); + + // set the nowTs to be the exact time when resources should be available again. + nowTs += waitInterval; + + // artificially go into the past to prove that when too early we should fail. + assertFalse(limiter.canExecute(nowTs - 500, lastTs)); + } + } + + @Test + public void testOverconsumption() { + RateLimiter limiter = new RateLimiter(); + limiter.set(10, TimeUnit.SECONDS); + + // 10 resources are available, but we need to consume 20 resources + // Verify that we have to wait at least 1.1sec to have 1 resource available + assertTrue(limiter.canExecute(0, 0)); + limiter.consume(20); + assertEquals(1100, limiter.waitInterval()); + + // Verify that after 1sec we need to wait for another 0.1sec to get a resource available + assertFalse(limiter.canExecute(1000, 0)); + assertEquals(100, limiter.waitInterval()); + + // Verify that after 1.1sec the resource is available + assertTrue(limiter.canExecute(1100, 0)); + assertEquals(0, limiter.waitInterval()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java index d2fc044..fb3dbe2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -2484,4 +2484,67 @@ public class TestAccessController extends SecureTestUtil { verifyAllowed(replicateLogEntriesAction, SUPERUSER, USER_ADMIN); verifyDenied(replicateLogEntriesAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); } + + @Test + public void testSetQuota() throws Exception { + AccessTestAction setUserQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetUserQuota(ObserverContext.createAndPrepare(CP_ENV, null), + null, null); + return null; + } + }; + + AccessTestAction setUserTableQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetUserQuota(ObserverContext.createAndPrepare(CP_ENV, null), + null, TEST_TABLE.getTableName(), null); + return null; + } + }; + + AccessTestAction setUserNamespaceQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetUserQuota(ObserverContext.createAndPrepare(CP_ENV, null), + null, (String)null, null); + return null; + } + }; + + AccessTestAction setTableQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetTableQuota(ObserverContext.createAndPrepare(CP_ENV, null), + TEST_TABLE.getTableName(), null); + return null; + } + }; + + AccessTestAction setNamespaceQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetNamespaceQuota(ObserverContext.createAndPrepare(CP_ENV, null), + null, null); + return null; + } + }; + + verifyAllowed(setUserQuotaAction, SUPERUSER, USER_ADMIN); + verifyDenied(setUserQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + + verifyAllowed(setUserTableQuotaAction, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(setUserTableQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); + + verifyAllowed(setUserNamespaceQuotaAction, SUPERUSER, USER_ADMIN); + verifyDenied(setUserNamespaceQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + + verifyAllowed(setTableQuotaAction, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(setTableQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); + + verifyAllowed(setNamespaceQuotaAction, SUPERUSER, USER_ADMIN); + verifyDenied(setNamespaceQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-shell/src/main/ruby/hbase.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase.rb b/hbase-shell/src/main/ruby/hbase.rb index d5fb5f6..281d18c 100644 --- a/hbase-shell/src/main/ruby/hbase.rb +++ b/hbase-shell/src/main/ruby/hbase.rb @@ -71,6 +71,12 @@ module HBaseConstants TABLE_CFS = 'TABLE_CFS' CONFIG = 'CONFIG' DATA = 'DATA' + USER = 'USER' + TABLE = 'TABLE' + NAMESPACE = 'NAMESPACE' + TYPE = 'TYPE' + NONE = 'NONE' + VALUE = 'VALUE' # Load constants from hbase java API def self.promote_constants(constants) @@ -90,6 +96,10 @@ end require 'hbase/hbase' require 'hbase/admin' require 'hbase/table' +require 'hbase/quotas' require 'hbase/replication_admin' require 'hbase/security' require 'hbase/visibility_labels' + + +include HBaseQuotasConstants \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-shell/src/main/ruby/hbase/hbase.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase/hbase.rb b/hbase-shell/src/main/ruby/hbase/hbase.rb index 89700a4..a17ece2 100644 --- a/hbase-shell/src/main/ruby/hbase/hbase.rb +++ b/hbase-shell/src/main/ruby/hbase/hbase.rb @@ -21,6 +21,7 @@ include Java require 'hbase/admin' require 'hbase/table' +require 'hbase/quotas' require 'hbase/security' require 'hbase/visibility_labels' @@ -62,6 +63,10 @@ module Hbase def visibility_labels_admin(formatter) ::Hbase::VisibilityLabelsAdmin.new(@connection.getAdmin, formatter) end + + def quotas_admin(formatter) + ::Hbase::QuotasAdmin.new(@connection.getAdmin, formatter) + end def shutdown @connection.close http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-shell/src/main/ruby/hbase/quotas.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb new file mode 100644 index 0000000..fa076a5 --- /dev/null +++ b/hbase-shell/src/main/ruby/hbase/quotas.rb @@ -0,0 +1,216 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include Java +java_import java.util.concurrent.TimeUnit +java_import org.apache.hadoop.hbase.TableName +java_import org.apache.hadoop.hbase.quotas.ThrottleType +java_import org.apache.hadoop.hbase.quotas.QuotaFilter +java_import org.apache.hadoop.hbase.quotas.QuotaRetriever +java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory + +module HBaseQuotasConstants + GLOBAL_BYPASS = 'GLOBAL_BYPASS' + THROTTLE_TYPE = 'THROTTLE_TYPE' + THROTTLE = 'THROTTLE' + REQUEST = 'REQUEST' +end + +module Hbase + class QuotasAdmin + def initialize(admin, formatter) + @admin = admin + @formatter = formatter + end + + def close + @admin.close + end + + def throttle(args) + raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash) + type = args.fetch(THROTTLE_TYPE, REQUEST) + type, limit, time_unit = _parse_limit(args.delete(LIMIT), ThrottleType, type) + if args.has_key?(USER) + user = args.delete(USER) + if args.has_key?(TABLE) + table = TableName.valueOf(args.delete(TABLE)) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.throttleUser(user, table, type, limit, time_unit) + elsif args.has_key?(NAMESPACE) + namespace = args.delete(NAMESPACE) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.throttleUser(user, namespace, type, limit, time_unit) + else + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.throttleUser(user, type, limit, time_unit) + end + elsif args.has_key?(TABLE) + table = TableName.valueOf(args.delete(TABLE)) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.throttleTable(table, type, limit, time_unit) + elsif args.has_key?(NAMESPACE) + namespace = args.delete(NAMESPACE) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.throttleNamespace(namespace, type, limit, time_unit) + else + raise "One of USER, TABLE or NAMESPACE must be specified" + end + @admin.setQuota(settings) + end + + def unthrottle(args) + raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash) + if args.has_key?(USER) + user = args.delete(USER) + if args.has_key?(TABLE) + table = TableName.valueOf(args.delete(TABLE)) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.unthrottleUser(user, table) + elsif args.has_key?(NAMESPACE) + namespace = args.delete(NAMESPACE) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.unthrottleUser(user, namespace) + else + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.unthrottleUser(user) + end + elsif args.has_key?(TABLE) + table = TableName.valueOf(args.delete(TABLE)) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.unthrottleTable(table) + elsif args.has_key?(NAMESPACE) + namespace = args.delete(NAMESPACE) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.unthrottleNamespace(namespace) + else + raise "One of USER, TABLE or NAMESPACE must be specified" + end + @admin.setQuota(settings) + end + + def set_global_bypass(bypass, args) + raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash) + + if args.has_key?(USER) + user = args.delete(USER) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.bypassGlobals(user, bypass) + else + raise "Expected USER" + end + @admin.setQuota(settings) + end + + def list_quotas(args = {}) + raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash) + + limit = args.delete("LIMIT") || -1 + count = 0 + + filter = QuotaFilter.new() + filter.setUserFilter(args.delete(USER)) if args.has_key?(USER) + filter.setTableFilter(args.delete(TABLE)) if args.has_key?(TABLE) + filter.setNamespaceFilter(args.delete(NAMESPACE)) if args.has_key?(NAMESPACE) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + + # Start the scanner + scanner = @admin.getQuotaRetriever(filter) + begin + iter = scanner.iterator + + # Iterate results + while iter.hasNext + if limit > 0 && count >= limit + break + end + + settings = iter.next + owner = { + USER => settings.getUserName(), + TABLE => settings.getTableName(), + NAMESPACE => settings.getNamespace(), + }.delete_if { |k, v| v.nil? }.map {|k, v| k.to_s + " => " + v.to_s} * ', ' + + yield owner, settings.to_s + + count += 1 + end + ensure + scanner.close() + end + + return count + end + + def _parse_size(str_limit) + str_limit = str_limit.downcase + match = /(\d+)([bkmgtp%]*)/.match(str_limit) + if match + if match[2] == '%' + return match[1].to_i + else + return _size_from_str(match[1].to_i, match[2]) + end + else + raise "Invalid size limit syntax" + end + end + + def _parse_limit(str_limit, type_cls, type) + str_limit = str_limit.downcase + match = /(\d+)(req|[bkmgtp])\/(sec|min|hour|day)/.match(str_limit) + if match + if match[2] == 'req' + limit = match[1].to_i + type = type_cls.valueOf(type + "_NUMBER") + else + limit = _size_from_str(match[1].to_i, match[2]) + type = type_cls.valueOf(type + "_SIZE") + end + + if limit <= 0 + raise "Invalid throttle limit, must be greater then 0" + end + + case match[3] + when 'sec' then time_unit = TimeUnit::SECONDS + when 'min' then time_unit = TimeUnit::MINUTES + when 'hour' then time_unit = TimeUnit::HOURS + when 'day' then time_unit = TimeUnit::DAYS + end + + return type, limit, time_unit + else + raise "Invalid throttle limit syntax" + end + end + + def _size_from_str(value, suffix) + case suffix + when 'k' then value <<= 10 + when 'm' then value <<= 20 + when 'g' then value <<= 30 + when 't' then value <<= 40 + when 'p' then value <<= 50 + end + return value + end + end +end \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-shell/src/main/ruby/shell.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 84646c2..1566685 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -99,6 +99,10 @@ module Shell def hbase_visibility_labels_admin @hbase_visibility_labels_admin ||= hbase.visibility_labels_admin(formatter) end + + def hbase_quotas_admin + @hbase_quotas_admin ||= hbase.quotas_admin(formatter) + end def export_commands(where) ::Shell.commands.keys.each do |cmd| @@ -372,6 +376,15 @@ Shell.load_command_group( ) Shell.load_command_group( + 'quotas', + :full_name => 'CLUSTER QUOTAS TOOLS', + :commands => %w[ + set_quota + list_quotas + ] +) + +Shell.load_command_group( 'security', :full_name => 'SECURITY TOOLS', :comment => "NOTE: Above commands are only applicable if running with the AccessController coprocessor", http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-shell/src/main/ruby/shell/commands.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands.rb b/hbase-shell/src/main/ruby/shell/commands.rb index 1e7ad6d..527e30f 100644 --- a/hbase-shell/src/main/ruby/shell/commands.rb +++ b/hbase-shell/src/main/ruby/shell/commands.rb @@ -65,6 +65,10 @@ module Shell def visibility_labels_admin @shell.hbase_visibility_labels_admin end + + def quotas_admin + @shell.hbase_quotas_admin + end #---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-shell/src/main/ruby/shell/commands/list_quotas.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/list_quotas.rb b/hbase-shell/src/main/ruby/shell/commands/list_quotas.rb new file mode 100644 index 0000000..682bb71 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/list_quotas.rb @@ -0,0 +1,52 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListQuotas < Command + def help + return <<-EOF +List the quota settings added to the system. +You can filter the result based on USER, TABLE, or NAMESPACE. + +For example: + + hbase> list_quotas + hbase> list_quotas USER => 'bob.*' + hbase> list_quotas USER => 'bob.*', TABLE => 't1' + hbase> list_quotas USER => 'bob.*', NAMESPACE => 'ns.*' + hbase> list_quotas TABLE => 'myTable' + hbase> list_quotas NAMESPACE => 'ns.*' +EOF + end + + def command(args = {}) + now = Time.now + formatter.header(["OWNER", "QUOTAS"]) + + #actually do the scanning + count = quotas_admin.list_quotas(args) do |row, cells| + formatter.row([ row, cells ]) + end + + formatter.footer(now, count) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/c031d8de/hbase-shell/src/main/ruby/shell/commands/set_quota.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb new file mode 100644 index 0000000..40e8a10 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb @@ -0,0 +1,70 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class SetQuota < Command + def help + return <<-EOF +Set a quota for a user, table, or namespace. +Syntax : set_quota TYPE => <type>, <args> + +TYPE => THROTTLE +The request limit can be expressed using the form 100req/sec, 100req/min +and the size limit can be expressed using the form 100k/sec, 100M/min +with (B, K, M, G, T, P) as valid size unit and (sec, min, hour, day) as valid time unit. +Currently the throttle limit is per machine - a limit of 100req/min +means that each machine can execute 100req/min. + +For example: + + hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => '10req/sec' + hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => '10M/sec' + hbase> set_quota TYPE => THROTTLE, USER => 'u1', TABLE => 't2', LIMIT => '5K/min' + hbase> set_quota TYPE => THROTTLE, USER => 'u1', NAMESPACE => 'ns2', LIMIT => NONE + hbase> set_quota TYPE => THROTTLE, NAMESPACE => 'ns1', LIMIT => '10req/sec' + hbase> set_quota TYPE => THROTTLE, TABLE => 't1', LIMIT => '10M/sec' + hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => NONE + hbase> set_quota USER => 'u1', GLOBAL_BYPASS => true +EOF + end + + def command(args = {}) + if args.has_key?(TYPE) + qtype = args.delete(TYPE) + case qtype + when THROTTLE + if args[LIMIT].eql? NONE + args.delete(LIMIT) + quotas_admin.unthrottle(args) + else + quotas_admin.throttle(args) + end + else + raise "Invalid TYPE argument. got " + qtype + end + elsif args.has_key?(GLOBAL_BYPASS) + quotas_admin.set_global_bypass(args.delete(GLOBAL_BYPASS), args) + else + raise "Expected TYPE argument" + end + end + end + end +end
