vongosling closed pull request #149: [ROCKETMQ-136]Provide a handy message queue producer for order message sharding URL: https://github.com/apache/rocketmq/pull/149
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/OrderMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/OrderMQProducer.java new file mode 100644 index 000000000..a8ba5b09c --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/OrderMQProducer.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.producer; + +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.selector.SelectMessageQueueByConsistentHash; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingException; + +/** + * A simple Order message Producer Facade to send order messages, which is a subclass of DefaultMQProducer <br> + * + * The messages with the same value returned from method {@code shardingKey} will try to be sent to the same message + * queue, user must extend this class and implement the abstract method shardingKey. + * + * <p> Please notice that this can not be guaranteed when the numbers of messages are changed.<br> When the number of + * message queues change before all the messages with the same sharding key sent successfully, the messages queue picked + * to sent to may be changed.<br> However, only as few as messages will be influenced thanks to the default selector + * backed by consistent hash algorithm. </p> + */ +public abstract class OrderMQProducer extends DefaultMQProducer { + private MessageQueueSelector selector = new SelectMessageQueueByConsistentHash(); + + public OrderMQProducer() { + } + + /** + * Constructor specifying producer group. + * + * @param producerGroup Producer group, see the name-sake field. + */ + public OrderMQProducer(String producerGroup) { + super(producerGroup); + } + + /** + * Constructor specifying both producer group and RPC hook. + * + * @param producerGroup Producer group, see the name-sake field. + * @param rpcHook RPC hook to execute per each remoting command execution. + */ + public OrderMQProducer(final String producerGroup, RPCHook rpcHook) { + super(producerGroup, rpcHook); + } + + /** + * Constructor specifying the RPC hook. + * + * @param rpcHook RPC hook to execute per each remoting command execution. + */ + public OrderMQProducer(RPCHook rpcHook) { + this(MixAll.DEFAULT_PRODUCER_GROUP, rpcHook); + } + + public MessageQueueSelector getSelector() { + return selector; + } + + public void setSelector(MessageQueueSelector selector) { + this.selector = selector; + } + + /** + * Callback method to return a unique sharding key for each messages. The sharding key will be used to pick a + * message queue. The messages with the same message queue will be sent to the same message queues. + * + * @param msg the message which is ready to send + * @return the sharding key which will be used to select message queue + */ + abstract String shardingKey(Message msg); + + @Override + public SendResult send(Message msg) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return super.send(msg, selector, shardingKey(msg)); + } + + @Override + public SendResult send(Message msg, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return super.send(msg, selector, shardingKey(msg), timeout); + } + + @Override + public void send(Message msg, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + super.send(msg, selector, shardingKey(msg), sendCallback); + } + + @Override + public void send(Message msg, SendCallback sendCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + super.send(msg, selector, shardingKey(msg), sendCallback, timeout); + } + + @Override + public void sendOneway(Message msg) + throws MQClientException, RemotingException, InterruptedException { + super.sendOneway(msg, selector, shardingKey(msg)); + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByConsistentHash.java b/client/src/main/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByConsistentHash.java new file mode 100644 index 000000000..f9d00153b --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByConsistentHash.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.producer.selector; + +import org.apache.rocketmq.client.producer.MessageQueueSelector; +import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter; +import org.apache.rocketmq.common.consistenthash.Node; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; + +import java.util.ArrayList; +import java.util.List; + +public class SelectMessageQueueByConsistentHash implements MessageQueueSelector { + + private final ConsistentHashRouter<MQNode> consistentHashRouter = new ConsistentHashRouter<MQNode>(null, 0); + private volatile List<MessageQueue> lastMqs = new ArrayList<MessageQueue>(); + private final int virtualNodeNum; + + public SelectMessageQueueByConsistentHash() { + this.virtualNodeNum = 10; + } + + public SelectMessageQueueByConsistentHash(int virtualNodeNum) { + this.virtualNodeNum = virtualNodeNum; + } + + @Override + public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) { + if (arg == null) { + throw new NullPointerException(); + } + synchronized (consistentHashRouter) { + rehash(mqs); + return consistentHashRouter.routeNode(arg.toString()).mq; + } + } + + private void rehash(List<MessageQueue> mqs) { + //newly added + List<MQNode> newMq = new ArrayList<MQNode>(); + for (MessageQueue mq : mqs) { + if (!lastMqs.contains(mq)) { + newMq.add(new MQNode(mq)); + } + } + + //remove already + List<MQNode> removeMq = new ArrayList<MQNode>(); + for (MessageQueue last : lastMqs) { + if (!mqs.contains(last)) { + removeMq.add(new MQNode(last)); + } + } + + for (MQNode mqNode : newMq) { + consistentHashRouter.addNode(mqNode, virtualNodeNum); + } + + for (MQNode mqNode : removeMq) { + consistentHashRouter.removeNode(mqNode); + } + + this.lastMqs = mqs; + + } + + class MQNode implements Node { + private final MessageQueue mq; + + public MQNode(MessageQueue mq) { + this.mq = mq; + } + + @Override + public String getKey() { + return mq.toString(); + } + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/OrderMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/OrderMQProducerTest.java new file mode 100644 index 000000000..15f8d60a5 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/producer/OrderMQProducerTest.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.producer; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.hook.SendMessageHook; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.lang.reflect.Field; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class OrderMQProducerTest { + @Spy + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + @Mock + private MQClientAPIImpl mQClientAPIImpl; + + private OrderMQProducer producer; + private String topic = "FooBar"; + private String producerGroupPrefix = "FooBar_PID"; + private ConcurrentHashMap<String /*batch*/, Set<Integer>> sentMQMap; + + final int orderMsgBatches = 1000; + final int numEachBatch = 10; + final CountDownLatch countDownLatch = new CountDownLatch( orderMsgBatches * numEachBatch); + + @Before + public void init() throws Exception { + String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); + producer = new OrderMQProducer(producerGroupTemp) { + @Override + String shardingKey(Message msg) { + String batch = msg.getUserProperty("batchId"); + return batch; + } + }; + + producer.setNamesrvAddr("127.0.0.1:9876"); + sentMQMap = new ConcurrentHashMap<String,Set <Integer>>(); + + producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageHook() { + @Override + public String hookName() { + return "check"; + } + + @Override + public void sendMessageBefore(SendMessageContext context) { + String batch = context.getMessage().getProperty("batchId"); + int queueId = context.getMq().getQueueId(); + sentMQMap.putIfAbsent(batch, Collections.synchronizedSet(new HashSet<Integer>())); + sentMQMap.get(batch).add(queueId); + countDownLatch.countDown(); + } + + @Override + public void sendMessageAfter(SendMessageContext context) { + + } + }); + producer.start(); + + Field field = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(producer.getDefaultMQProducerImpl(), mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl()); + + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(createSendResult(SendStatus.SEND_OK)); + } + + @After + public void terminate() { + producer.shutdown(); + } + + + @Test + public void testBasic() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + ExecutorService es = Executors.newFixedThreadPool(5); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute(8)); + mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + + for (int i = 0 ; i <orderMsgBatches ;i ++) {// batches + for (int j = 0 ; j <numEachBatch; j++) {// several messages for each batch + final int batchIndex = i; + es.submit(new Runnable() { + @Override public void run() { + try { + final Message msg = new Message(topic, (batchIndex+"").getBytes()); + msg.putUserProperty("batchId","batch"+batchIndex); + SendResult sendResult = producer.send(msg); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + + } + } + + countDownLatch.await(10, TimeUnit.SECONDS); + es.shutdown(); + assertThat(countDownLatch.getCount()).isEqualTo(0); + for (String sendBatch : sentMQMap.keySet()) { + assertThat(sentMQMap.get(sendBatch).size()).isEqualTo(1);// only one mq is sent for each batch, meaning that they are all sent to one batches + } + + } + + @Test + public void testQueueClose() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute(8)); + + for (int i = 0 ; i <orderMsgBatches ;i ++) { + final Message msg = new Message(topic, (i+"").getBytes()); + msg.putUserProperty("batchId","influence-batch"+i); + SendResult sendResult = producer.send(msg); + } + + //one queue is closed after 100 ms + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute(7)); + + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + scheduledExecutorService.schedule(new Runnable() { + @Override public void run() { + mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + } + },10,TimeUnit.MILLISECONDS); + + //send the rest after q changes + for (int i = 0 ; i <orderMsgBatches ; i++) { + for (int j = 1; j < numEachBatch ; j++) { + final Message msg = new Message(topic, (i + "").getBytes()); + msg.putUserProperty("batchId", "influence-batch" + i); + SendResult sendResult = producer.send(msg); + } + } + + scheduledExecutorService.shutdown(); + for (String sendBatch : sentMQMap.keySet()) { + if (!sentMQMap.get(sendBatch).contains(7)) {//the messages which are sent to the existing mq will not be influenced + assertThat(sentMQMap.get(sendBatch).size()).isEqualTo(1); + } else { + assertThat(sentMQMap.get(sendBatch).size()).isLessThanOrEqualTo(2); + } + } + } + + @Test + public void testQueueScale() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute(8)); + + final int orderMsgBatches = 1000; + final int numEachBatch = 10; + + for (int i = 0 ; i <orderMsgBatches ;i ++) { + final Message msg = new Message(topic, (i+"").getBytes()); + msg.putUserProperty("batchId","influence-batch"+i); + SendResult sendResult = producer.send(msg); + } + + //2 queue is scaled + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute(9)); + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + scheduledExecutorService.schedule(new Runnable() { + @Override public void run() { + mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + } + },100,TimeUnit.MILLISECONDS); + + //send another rest after q changes + for (int i = 0 ; i <orderMsgBatches ;i ++) {// 1000 batches + for (int j=1; j <= numEachBatch ;j++) { + final Message msg = new Message(topic, (i + "").getBytes()); + msg.putUserProperty("batchId", "influence-batch" + i); + SendResult sendResult = producer.send(msg); + } + } + + scheduledExecutorService.shutdown(); + for (String sendBatch : sentMQMap.keySet()) { + if (!sentMQMap.get(sendBatch).contains(8)) {//not be influenced if they are not sharded to new mq + assertThat(sentMQMap.get(sendBatch).size()).isEqualTo(1); + } else { + assertThat(sentMQMap.get(sendBatch).size()).isLessThanOrEqualTo(2); + } + } + } + + + public static TopicRouteData createTopicRoute(int queueNum) { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap<String, List<String>>()); + List<BrokerData> brokerDataList = new ArrayList<BrokerData>(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("BrokerA"); + brokerData.setCluster("DefaultCluster"); + HashMap<Long, String> brokerAddrs = new HashMap<Long, String>(); + brokerAddrs.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List<QueueData> queueDataList = new ArrayList<QueueData>(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("BrokerA"); + queueData.setPerm(6); + queueData.setReadQueueNums(queueNum); + queueData.setWriteQueueNums(queueNum); + queueData.setTopicSynFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("123"); + sendResult.setOffsetMsgId("123"); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + return sendResult; + } + +} \ No newline at end of file ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
