This is an automated email from the ASF dual-hosted git repository.
penghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 523b5c85fe7 [Fix][broker]Fix topic dispatch rate limiter not init on
broker-level (#16084)
523b5c85fe7 is described below
commit 523b5c85fe78e25da5595b4a0dfa56f85e84e7bc
Author: Xiaoyu Hou <[email protected]>
AuthorDate: Mon Jul 4 10:33:50 2022 +0800
[Fix][broker]Fix topic dispatch rate limiter not init on broker-level
(#16084)
---
.../pulsar/broker/service/AbstractTopic.java | 3 +
.../pulsar/broker/service/BrokerService.java | 4 +-
.../broker/service/persistent/PersistentTopic.java | 23 ++--
.../service/TopicDispatchRateLimiterTest.java | 142 +++++++++++++++++++++
4 files changed, 154 insertions(+), 18 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
index 59d9d875fd2..0010c9bbd8a 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
@@ -894,6 +894,9 @@ public abstract class AbstractTopic implements Topic,
TopicPolicyListener<TopicP
}
}
+ public void updateDispatchRateLimiter() {
+ }
+
@Override
public void resetBrokerPublishCountAndEnableReadIfRequired(boolean
doneBrokerReset) {
// topic rate not exceeded, and completed broker limiter reset.
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index f56b0012b31..5e78417260b 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -2414,9 +2414,7 @@ public class BrokerService implements Closeable {
forEachTopic(topic -> {
if (topic instanceof AbstractTopic) {
((AbstractTopic) topic).updateBrokerDispatchRate();
- }
- if (topic.getDispatchRateLimiter().isPresent()) {
- topic.getDispatchRateLimiter().get().updateDispatchRate();
+ ((AbstractTopic) topic).updateDispatchRateLimiter();
}
});
});
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 61ff46e5b6d..d0aa9eea52f 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -2357,6 +2357,12 @@ public class PersistentTopic extends AbstractTopic
implements Topic, AddEntryCal
return checkPersistencePolicies();
}
+ @Override
+ public void updateDispatchRateLimiter() {
+ initializeDispatchRateLimiterIfNeeded();
+ dispatchRateLimiter.ifPresent(DispatchRateLimiter::updateDispatchRate);
+ }
+
@Override
public CompletableFuture<Void> onPoliciesUpdate(Policies data) {
if (log.isDebugEnabled()) {
@@ -2374,7 +2380,7 @@ public class PersistentTopic extends AbstractTopic
implements Topic, AddEntryCal
isAllowAutoUpdateSchema = data.is_allow_auto_update_schema;
- initializeDispatchRateLimiterIfNeeded();
+ updateDispatchRateLimiter();
updateSubscribeRateLimiter();
@@ -2393,8 +2399,6 @@ public class PersistentTopic extends AbstractTopic
implements Topic, AddEntryCal
CompletableFuture<Void> replicationFuture =
checkReplicationAndRetryOnFailure();
CompletableFuture<Void> dedupFuture =
checkDeduplicationStatus();
CompletableFuture<Void> persistentPoliciesFuture =
checkPersistencePolicies();
- // update rate-limiter if policies updated
-
dispatchRateLimiter.ifPresent(DispatchRateLimiter::updateDispatchRate);
return CompletableFuture.allOf(replicationFuture, dedupFuture,
persistentPoliciesFuture,
preCreateSubscriptionForCompactionIfNeeded());
});
@@ -3013,10 +3017,7 @@ public class PersistentTopic extends AbstractTopic
implements Topic, AddEntryCal
}
updateTopicPolicy(policies);
- initializeTopicDispatchRateLimiterIfNeeded(policies);
-
- dispatchRateLimiter.ifPresent(DispatchRateLimiter::updateDispatchRate);
-
+ updateDispatchRateLimiter();
updateSubscriptionsDispatcherRateLimiter().thenRun(() -> {
updatePublishDispatcher();
updateSubscribeRateLimiter();
@@ -3054,14 +3055,6 @@ public class PersistentTopic extends AbstractTopic
implements Topic, AddEntryCal
return FutureUtil.waitForAll(subscriptionCheckFutures);
}
- private void initializeTopicDispatchRateLimiterIfNeeded(TopicPolicies
policies) {
- synchronized (dispatchRateLimiterLock) {
- if (!dispatchRateLimiter.isPresent() && policies.getDispatchRate()
!= null) {
- this.dispatchRateLimiter = Optional.of(new
DispatchRateLimiter(this, Type.TOPIC));
- }
- }
- }
-
protected CompletableFuture<Void> initTopicPolicy() {
if (brokerService.pulsar().getConfig().isSystemTopicEnabled()
&&
brokerService.pulsar().getConfig().isTopicLevelPoliciesEnabled()) {
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicDispatchRateLimiterTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicDispatchRateLimiterTest.java
new file mode 100644
index 00000000000..5ff3c5ec9eb
--- /dev/null
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicDispatchRateLimiterTest.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import lombok.Cleanup;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.common.policies.data.DispatchRate;
+import org.awaitility.Awaitility;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class TopicDispatchRateLimiterTest extends BrokerTestBase {
+ @BeforeMethod
+ @Override
+ protected void setup() throws Exception {
+ conf.setDispatchThrottlingRatePerTopicInMsg(0);
+ conf.setDispatchThrottlingRatePerTopicInByte(0L);
+ super.baseSetup();
+ }
+
+ @AfterMethod(alwaysRun = true)
+ @Override
+ protected void cleanup() throws Exception {
+ super.internalCleanup();
+ }
+
+ @Test
+ public void testTopicDispatchRateLimiterPerTopicInMsgOnlyBrokerLevel()
throws Exception {
+ final String topicName = "persistent://" + newTopicName();
+
+ @Cleanup
+ Producer<byte[]> producer =
pulsarClient.newProducer().topic(topicName).create();
+ PersistentTopic topic = (PersistentTopic)
pulsar.getBrokerService().getOrCreateTopic(topicName).get();
+ assertNotNull(topic);
+ assertTrue(topic.getDispatchRateLimiter().isEmpty());
+
+
admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg",
"100");
+ Awaitility.await().untilAsserted(() ->
+
assertEquals(pulsar.getConfig().getDispatchThrottlingRatePerTopicInMsg(), 100));
+
+ Awaitility.await().untilAsserted(() ->
+ assertTrue(topic.getDispatchRateLimiter().isPresent()));
+
assertEquals(topic.getDispatchRateLimiter().get().getAvailableDispatchRateLimitOnMsg(),
100L);
+ }
+
+ @Test
+ public void testTopicDispatchRateLimiterPerTopicInByteOnlyBrokerLevel()
throws Exception {
+ final String topicName = "persistent://" + newTopicName();
+
+ @Cleanup
+ Producer<byte[]> producer =
pulsarClient.newProducer().topic(topicName).create();
+ PersistentTopic topic = (PersistentTopic)
pulsar.getBrokerService().getOrCreateTopic(topicName).get();
+ assertNotNull(topic);
+ assertTrue(topic.getDispatchRateLimiter().isEmpty());
+
+
admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInByte",
"1000");
+ Awaitility.await().untilAsserted(() ->
+
assertEquals(pulsar.getConfig().getDispatchThrottlingRatePerTopicInByte(),
1000L));
+
+ Awaitility.await().untilAsserted(() ->
+ assertTrue(topic.getDispatchRateLimiter().isPresent()));
+
assertEquals(topic.getDispatchRateLimiter().get().getAvailableDispatchRateLimitOnByte(),
1000L);
+ }
+
+ @Test
+ public void testTopicDispatchRateLimiterOnlyNamespaceLevel() throws
Exception {
+ final String topicName = "persistent://" + newTopicName();
+
+ @Cleanup
+ Producer<byte[]> producer =
pulsarClient.newProducer().topic(topicName).create();
+ PersistentTopic topic = (PersistentTopic)
pulsar.getBrokerService().getOrCreateTopic(topicName).get();
+ assertNotNull(topic);
+ assertTrue(topic.getDispatchRateLimiter().isEmpty());
+
+ DispatchRate dispatchRate = DispatchRate
+ .builder()
+ .dispatchThrottlingRateInMsg(100)
+ .dispatchThrottlingRateInByte(1000L)
+ .build();
+ admin.namespaces().setDispatchRate("prop/ns-abc", dispatchRate);
+
+ Awaitility.await().untilAsserted(() -> {
+ assertNotNull(admin.namespaces().getDispatchRate("prop/ns-abc"));
+
assertEquals(admin.namespaces().getDispatchRate("prop/ns-abc").getDispatchThrottlingRateInMsg(),
100);
+
assertEquals(admin.namespaces().getDispatchRate("prop/ns-abc").getDispatchThrottlingRateInByte(),
1000L);
+ });
+
+ Awaitility.await().untilAsserted(() ->
assertTrue(topic.getDispatchRateLimiter().isPresent()));
+
assertEquals(topic.getDispatchRateLimiter().get().getAvailableDispatchRateLimitOnMsg(),
100);
+
assertEquals(topic.getDispatchRateLimiter().get().getAvailableDispatchRateLimitOnByte(),
1000L);
+ }
+
+ @Test
+ public void testTopicDispatchRateLimiterOnlyTopicLevel() throws Exception {
+ final String topicName = "persistent://" + newTopicName();
+
+ @Cleanup
+ Producer<byte[]> producer =
pulsarClient.newProducer().topic(topicName).create();
+ PersistentTopic topic = (PersistentTopic)
pulsar.getBrokerService().getOrCreateTopic(topicName).get();
+ assertNotNull(topic);
+ assertTrue(topic.getDispatchRateLimiter().isEmpty());
+
+ DispatchRate dispatchRate = DispatchRate
+ .builder()
+ .dispatchThrottlingRateInMsg(100)
+ .dispatchThrottlingRateInByte(1000L)
+ .build();
+ admin.topicPolicies().setDispatchRate(topicName, dispatchRate);
+
+ Awaitility.await().untilAsserted(() -> {
+ assertNotNull(admin.topicPolicies().getDispatchRate(topicName));
+
assertEquals(admin.topicPolicies().getDispatchRate(topicName).getDispatchThrottlingRateInMsg(),
100);
+
assertEquals(admin.topicPolicies().getDispatchRate(topicName).getDispatchThrottlingRateInByte(),
1000L);
+ });
+
+ Awaitility.await().untilAsserted(() ->
assertTrue(topic.getDispatchRateLimiter().isPresent()));
+
assertEquals(topic.getDispatchRateLimiter().get().getAvailableDispatchRateLimitOnMsg(),
100);
+
assertEquals(topic.getDispatchRateLimiter().get().getAvailableDispatchRateLimitOnByte(),
1000L);
+ }
+}
