This is an automated email from the ASF dual-hosted git repository.
cconnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 8c6ecf9eceb HBASE-29479: QuotaCache should always return accurate
information (#7188)
8c6ecf9eceb is described below
commit 8c6ecf9eceb90dfb45ac60f63bfaa128a5cabc32
Author: Charles Connell <[email protected]>
AuthorDate: Wed Sep 3 13:52:45 2025 -0400
HBASE-29479: QuotaCache should always return accurate information (#7188)
Signed-off by: Ray Mattingly <[email protected]>
---
.../org/apache/hadoop/hbase/quotas/QuotaCache.java | 204 +++++++++++----------
.../hbase/quotas/TestDefaultAtomicQuota.java | 9 -
.../apache/hadoop/hbase/quotas/TestQuotaCache.java | 118 +++++++++++-
3 files changed, 222 insertions(+), 109 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
index c180ddd4c5e..fb1b6e4b0d9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hbase.quotas;
-import static org.apache.hadoop.hbase.util.ConcurrentMapUtils.computeIfAbsent;
-
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
@@ -30,6 +28,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterMetrics;
import org.apache.hadoop.hbase.ClusterMetrics.Option;
@@ -56,10 +55,7 @@ import
org.apache.hbase.thirdparty.com.google.common.cache.LoadingCache;
/**
* Cache that keeps track of the quota settings for the users and tables that
are interacting with
- * it. To avoid blocking the operations if the requested quota is not in cache
an "empty quota" will
- * be returned and the request to fetch the quota information will be enqueued
for the next refresh.
- * TODO: At the moment the Cache has a Chore that will be triggered every 5min
or on cache-miss
- * events. Later the Quotas will be pushed using the notification system.
+ * it.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -100,6 +96,57 @@ public class QuotaCache implements Stoppable {
private QuotaRefresherChore refreshChore;
private boolean stopped = true;
+ private final Fetcher<String, UserQuotaState> userQuotaStateFetcher = new
Fetcher<>() {
+ @Override
+ public Get makeGet(final String user) {
+ final Set<String> namespaces =
QuotaCache.this.namespaceQuotaCache.keySet();
+ final Set<TableName> tables = QuotaCache.this.tableQuotaCache.keySet();
+ return QuotaUtil.makeGetForUserQuotas(user, tables, namespaces);
+ }
+
+ @Override
+ public Map<String, UserQuotaState> fetchEntries(final List<Get> gets)
throws IOException {
+ return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), gets,
tableMachineQuotaFactors,
+ machineQuotaFactor);
+ }
+ };
+
+ private final Fetcher<String, QuotaState> regionServerQuotaStateFetcher =
new Fetcher<>() {
+ @Override
+ public Get makeGet(final String regionServer) {
+ return QuotaUtil.makeGetForRegionServerQuotas(regionServer);
+ }
+
+ @Override
+ public Map<String, QuotaState> fetchEntries(final List<Get> gets) throws
IOException {
+ return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection(),
gets);
+ }
+ };
+
+ private final Fetcher<TableName, QuotaState> tableQuotaStateFetcher = new
Fetcher<>() {
+ @Override
+ public Get makeGet(final TableName table) {
+ return QuotaUtil.makeGetForTableQuotas(table);
+ }
+
+ @Override
+ public Map<TableName, QuotaState> fetchEntries(final List<Get> gets)
throws IOException {
+ return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), gets,
tableMachineQuotaFactors);
+ }
+ };
+
+ private final Fetcher<String, QuotaState> namespaceQuotaStateFetcher = new
Fetcher<>() {
+ @Override
+ public Get makeGet(final String namespace) {
+ return QuotaUtil.makeGetForNamespaceQuotas(namespace);
+ }
+
+ @Override
+ public Map<String, QuotaState> fetchEntries(final List<Get> gets) throws
IOException {
+ return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), gets,
machineQuotaFactor);
+ }
+ };
+
public QuotaCache(final RegionServerServices rsServices) {
this.rsServices = rsServices;
this.userOverrideRequestAttributeKey =
@@ -153,8 +200,13 @@ public class QuotaCache implements Stoppable {
* @return the quota info associated to specified user
*/
public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) {
- return computeIfAbsent(userQuotaCache, getQuotaUserName(ugi),
- () ->
QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration(), 0L));
+ String user = getQuotaUserName(ugi);
+ if (!userQuotaCache.containsKey(user)) {
+ userQuotaCache.put(user,
+ QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration(),
0L));
+ fetch("user", userQuotaCache, userQuotaStateFetcher);
+ }
+ return userQuotaCache.get(user);
}
/**
@@ -163,7 +215,11 @@ public class QuotaCache implements Stoppable {
* @return the limiter associated to the specified table
*/
public QuotaLimiter getTableLimiter(final TableName table) {
- return getQuotaState(this.tableQuotaCache, table).getGlobalLimiter();
+ if (!tableQuotaCache.containsKey(table)) {
+ tableQuotaCache.put(table, new QuotaState());
+ fetch("table", tableQuotaCache, tableQuotaStateFetcher);
+ }
+ return tableQuotaCache.get(table).getGlobalLimiter();
}
/**
@@ -172,7 +228,11 @@ public class QuotaCache implements Stoppable {
* @return the limiter associated to the specified namespace
*/
public QuotaLimiter getNamespaceLimiter(final String namespace) {
- return getQuotaState(this.namespaceQuotaCache,
namespace).getGlobalLimiter();
+ if (!namespaceQuotaCache.containsKey(namespace)) {
+ namespaceQuotaCache.put(namespace, new QuotaState());
+ fetch("namespace", namespaceQuotaCache, namespaceQuotaStateFetcher);
+ }
+ return namespaceQuotaCache.get(namespace).getGlobalLimiter();
}
/**
@@ -181,13 +241,41 @@ public class QuotaCache implements Stoppable {
* @return the limiter associated to the specified region server
*/
public QuotaLimiter getRegionServerQuotaLimiter(final String regionServer) {
- return getQuotaState(this.regionServerQuotaCache,
regionServer).getGlobalLimiter();
+ if (!regionServerQuotaCache.containsKey(regionServer)) {
+ regionServerQuotaCache.put(regionServer, new QuotaState());
+ fetch("regionServer", regionServerQuotaCache,
regionServerQuotaStateFetcher);
+ }
+ return regionServerQuotaCache.get(regionServer).getGlobalLimiter();
}
protected boolean isExceedThrottleQuotaEnabled() {
return exceedThrottleQuotaEnabled;
}
+ private <K, V extends QuotaState> void fetch(final String type, final Map<K,
V> quotasMap,
+ final Fetcher<K, V> fetcher) {
+ // Find the quota entries to update
+ List<Get> gets =
quotasMap.keySet().stream().map(fetcher::makeGet).collect(Collectors.toList());
+
+ // fetch and update the quota entries
+ if (!gets.isEmpty()) {
+ try {
+ for (Map.Entry<K, V> entry : fetcher.fetchEntries(gets).entrySet()) {
+ V quotaInfo = quotasMap.putIfAbsent(entry.getKey(),
entry.getValue());
+ if (quotaInfo != null) {
+ quotaInfo.update(entry.getValue());
+ }
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Loading {} key={} quotas={}", type, entry.getKey(),
quotaInfo);
+ }
+ }
+ } catch (IOException e) {
+ LOG.warn("Unable to read {} from quota table", type, e);
+ }
+ }
+ }
+
/**
* Applies a request attribute user override if available, otherwise returns
the UGI's short
* username
@@ -210,14 +298,6 @@ public class QuotaCache implements Stoppable {
return Bytes.toString(override);
}
- /**
- * Returns the QuotaState requested. If the quota info is not in cache an
empty one will be
- * returned and the quota request will be enqueued for the next cache
refresh.
- */
- private <K> QuotaState getQuotaState(final ConcurrentMap<K, QuotaState>
quotasMap, final K key) {
- return computeIfAbsent(quotasMap, key, QuotaState::new);
- }
-
void triggerCacheRefresh() {
refreshChore.triggerNow();
}
@@ -226,10 +306,6 @@ public class QuotaCache implements Stoppable {
refreshChore.chore();
}
- long getLastUpdate() {
- return refreshChore.lastUpdate;
- }
-
Map<String, QuotaState> getNamespaceQuotaCache() {
return namespaceQuotaCache;
}
@@ -248,8 +324,6 @@ public class QuotaCache implements Stoppable {
// TODO: Remove this once we have the notification bus
private class QuotaRefresherChore extends ScheduledChore {
- private long lastUpdate = 0;
-
// Querying cluster metrics so often, per-RegionServer, limits horizontal
scalability.
// So we cache the results to reduce that load.
private final RefreshableExpiringValueCache<ClusterMetrics>
tableRegionStatesClusterMetrics;
@@ -307,74 +381,12 @@ public class QuotaCache implements Stoppable {
.computeIfAbsent(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, key ->
new QuotaState());
updateQuotaFactors();
- fetchNamespaceQuotaState();
- fetchTableQuotaState();
- fetchUserQuotaState();
- fetchRegionServerQuotaState();
+ fetchAndEvict("namespace", QuotaCache.this.namespaceQuotaCache,
namespaceQuotaStateFetcher);
+ fetchAndEvict("table", QuotaCache.this.tableQuotaCache,
tableQuotaStateFetcher);
+ fetchAndEvict("user", QuotaCache.this.userQuotaCache,
userQuotaStateFetcher);
+ fetchAndEvict("regionServer", QuotaCache.this.regionServerQuotaCache,
+ regionServerQuotaStateFetcher);
fetchExceedThrottleQuota();
- lastUpdate = EnvironmentEdgeManager.currentTime();
- }
-
- private void fetchNamespaceQuotaState() {
- fetch("namespace", QuotaCache.this.namespaceQuotaCache, new
Fetcher<String, QuotaState>() {
- @Override
- public Get makeGet(final Map.Entry<String, QuotaState> entry) {
- return QuotaUtil.makeGetForNamespaceQuotas(entry.getKey());
- }
-
- @Override
- public Map<String, QuotaState> fetchEntries(final List<Get> gets)
throws IOException {
- return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(),
gets,
- machineQuotaFactor);
- }
- });
- }
-
- private void fetchTableQuotaState() {
- fetch("table", QuotaCache.this.tableQuotaCache, new Fetcher<TableName,
QuotaState>() {
- @Override
- public Get makeGet(final Map.Entry<TableName, QuotaState> entry) {
- return QuotaUtil.makeGetForTableQuotas(entry.getKey());
- }
-
- @Override
- public Map<TableName, QuotaState> fetchEntries(final List<Get> gets)
throws IOException {
- return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), gets,
- tableMachineQuotaFactors);
- }
- });
- }
-
- private void fetchUserQuotaState() {
- final Set<String> namespaces =
QuotaCache.this.namespaceQuotaCache.keySet();
- final Set<TableName> tables = QuotaCache.this.tableQuotaCache.keySet();
- fetch("user", QuotaCache.this.userQuotaCache, new Fetcher<String,
UserQuotaState>() {
- @Override
- public Get makeGet(final Map.Entry<String, UserQuotaState> entry) {
- return QuotaUtil.makeGetForUserQuotas(entry.getKey(), tables,
namespaces);
- }
-
- @Override
- public Map<String, UserQuotaState> fetchEntries(final List<Get> gets)
throws IOException {
- return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), gets,
- tableMachineQuotaFactors, machineQuotaFactor);
- }
- });
- }
-
- private void fetchRegionServerQuotaState() {
- fetch("regionServer", QuotaCache.this.regionServerQuotaCache,
- new Fetcher<String, QuotaState>() {
- @Override
- public Get makeGet(final Map.Entry<String, QuotaState> entry) {
- return QuotaUtil.makeGetForRegionServerQuotas(entry.getKey());
- }
-
- @Override
- public Map<String, QuotaState> fetchEntries(final List<Get> gets)
throws IOException {
- return
QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection(), gets);
- }
- });
}
private void fetchExceedThrottleQuota() {
@@ -386,7 +398,7 @@ public class QuotaCache implements Stoppable {
}
}
- private <K, V extends QuotaState> void fetch(final String type,
+ private <K, V extends QuotaState> void fetchAndEvict(final String type,
final ConcurrentMap<K, V> quotasMap, final Fetcher<K, V> fetcher) {
long now = EnvironmentEdgeManager.currentTime();
long evictPeriod = getPeriod() * EVICT_PERIOD_FACTOR;
@@ -398,7 +410,7 @@ public class QuotaCache implements Stoppable {
if (lastQuery > 0 && (now - lastQuery) >= evictPeriod) {
toRemove.add(entry.getKey());
} else {
- gets.add(fetcher.makeGet(entry));
+ gets.add(fetcher.makeGet(entry.getKey()));
}
}
@@ -543,8 +555,8 @@ public class QuotaCache implements Stoppable {
T get() throws Exception;
}
- static interface Fetcher<Key, Value> {
- Get makeGet(Map.Entry<Key, Value> entry);
+ interface Fetcher<Key, Value> {
+ Get makeGet(Key key);
Map<Key, Value> fetchEntries(List<Get> gets) throws IOException;
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java
index 383068259a4..87e3f7f332b 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java
@@ -81,10 +81,6 @@ public class TestDefaultAtomicQuota {
@Test
public void testDefaultAtomicReadLimits() throws Exception {
- // No write throttling
- configureLenientThrottle(ThrottleType.ATOMIC_WRITE_SIZE);
- refreshQuotas();
-
// Should have a strict throttle by default
TEST_UTIL.waitFor(60_000, () -> runIncTest(100) < 100);
@@ -102,11 +98,6 @@ public class TestDefaultAtomicQuota {
@Test
public void testDefaultAtomicWriteLimits() throws Exception {
- // No read throttling
- configureLenientThrottle(ThrottleType.ATOMIC_REQUEST_NUMBER);
- configureLenientThrottle(ThrottleType.ATOMIC_READ_SIZE);
- refreshQuotas();
-
// Should have a strict throttle by default
TEST_UTIL.waitFor(60_000, () -> runIncTest(100) < 100);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
index 9dcf79cd574..0f2e648d7c4 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
@@ -20,13 +20,16 @@ package org.apache.hadoop.hbase.quotas;
import static
org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.waitMinuteQuota;
import static org.junit.Assert.assertEquals;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.After;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
@@ -42,8 +45,8 @@ public class TestQuotaCache {
private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
private static final int REFRESH_TIME_MS = 1000;
- @After
- public void tearDown() throws Exception {
+ @AfterClass
+ public static void tearDown() throws Exception {
ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL);
EnvironmentEdgeManager.reset();
TEST_UTIL.shutdownMiniCluster();
@@ -68,7 +71,6 @@ public class TestQuotaCache {
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
UserQuotaState userQuotaState = quotaCache.getUserQuotaState(ugi);
- assertEquals(userQuotaState.getLastUpdate(), 0);
QuotaCache.TEST_BLOCK_REFRESH = false;
// new user should have refreshed immediately
@@ -86,4 +88,112 @@ public class TestQuotaCache {
// should refresh after time has passed
TEST_UTIL.waitFor(5_000, () -> lastUpdate !=
userQuotaState.getLastUpdate());
}
+
+ @Test
+ public void testUserQuotaLookup() throws Exception {
+ QuotaCache quotaCache =
+ ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get();
+ final Admin admin = TEST_UTIL.getAdmin();
+ admin.setQuota(QuotaSettingsFactory.throttleUser("my_user",
ThrottleType.READ_NUMBER, 3737,
+ TimeUnit.MINUTES));
+
+ // Setting a quota and then looking it up from the cache should work, even
if the cache has not
+ // refreshed
+ UserGroupInformation ugi =
UserGroupInformation.createRemoteUser("my_user");
+ QuotaLimiter quotaLimiter = quotaCache.getUserLimiter(ugi,
TableName.valueOf("my_table"));
+ assertEquals(3737, quotaLimiter.getReadNumLimit());
+
+ // if no specific user quota, fall back to default
+ ugi = UserGroupInformation.createRemoteUser("my_user2");
+ quotaLimiter = quotaCache.getUserLimiter(ugi,
TableName.valueOf("my_table"));
+ assertEquals(1000, quotaLimiter.getReadNumLimit());
+
+ // still works after refresh
+ quotaCache.forceSynchronousCacheRefresh();
+ ugi = UserGroupInformation.createRemoteUser("my_user");
+ quotaLimiter = quotaCache.getUserLimiter(ugi,
TableName.valueOf("my_table"));
+ assertEquals(3737, quotaLimiter.getReadNumLimit());
+
+ ugi = UserGroupInformation.createRemoteUser("my_user2");
+ quotaLimiter = quotaCache.getUserLimiter(ugi,
TableName.valueOf("my_table"));
+ assertEquals(1000, quotaLimiter.getReadNumLimit());
+ }
+
+ @Test
+ public void testTableQuotaLookup() throws Exception {
+ QuotaCache quotaCache =
+ ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get();
+ final Admin admin = TEST_UTIL.getAdmin();
+
admin.setQuota(QuotaSettingsFactory.throttleTable(TableName.valueOf("my_table"),
+ ThrottleType.READ_NUMBER, 3737, TimeUnit.MINUTES));
+
+ // Setting a quota and then looking it up from the cache should work, even
if the cache has not
+ // refreshed
+ QuotaLimiter quotaLimiter =
quotaCache.getTableLimiter(TableName.valueOf("my_table"));
+ assertEquals(3737, quotaLimiter.getReadNumLimit());
+
+ // if no specific table quota, fall back to default
+ quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table2"));
+ assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit());
+
+ // still works after refresh
+ quotaCache.forceSynchronousCacheRefresh();
+ quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table"));
+ assertEquals(3737, quotaLimiter.getReadNumLimit());
+
+ quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table2"));
+ assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit());
+ }
+
+ @Test
+ public void testNamespaceQuotaLookup() throws Exception {
+ QuotaCache quotaCache =
+ ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get();
+ final Admin admin = TEST_UTIL.getAdmin();
+ admin.setQuota(QuotaSettingsFactory.throttleNamespace("my_namespace",
ThrottleType.READ_NUMBER,
+ 3737, TimeUnit.MINUTES));
+
+ // Setting a quota and then looking it up from the cache should work, even
if the cache has not
+ // refreshed
+ QuotaLimiter quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace");
+ assertEquals(3737, quotaLimiter.getReadNumLimit());
+
+ // if no specific namespace quota, fall back to default
+ quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace2");
+ assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit());
+
+ // still works after refresh
+ quotaCache.forceSynchronousCacheRefresh();
+ quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace");
+ assertEquals(3737, quotaLimiter.getReadNumLimit());
+
+ quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace2");
+ assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit());
+ }
+
+ @Test
+ public void testRegionServerQuotaLookup() throws Exception {
+ QuotaCache quotaCache =
+ ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get();
+ final Admin admin = TEST_UTIL.getAdmin();
+
admin.setQuota(QuotaSettingsFactory.throttleRegionServer("my_region_server",
+ ThrottleType.READ_NUMBER, 3737, TimeUnit.MINUTES));
+
+ // Setting a quota and then looking it up from the cache should work, even
if the cache has not
+ // refreshed
+ QuotaLimiter quotaLimiter =
quotaCache.getRegionServerQuotaLimiter("my_region_server");
+ assertEquals(3737, quotaLimiter.getReadNumLimit());
+
+ // if no specific server quota, fall back to default
+ quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server2");
+ assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit());
+
+ // still works after refresh
+ quotaCache.forceSynchronousCacheRefresh();
+ quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server");
+ assertEquals(3737, quotaLimiter.getReadNumLimit());
+
+ quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server2");
+ assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit());
+ }
}
