drpmma commented on code in PR #5887:
URL: https://github.com/apache/rocketmq/pull/5887#discussion_r1102517850
##########
store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java:
##########
@@ -2743,6 +2835,282 @@ public String getServiceName() {
}
+ class MainBatchDispatchRequestService extends ServiceThread {
+
+ private final ExecutorService batchDispatchRequestExecutor;
+
+ public MainBatchDispatchRequestService() {
+ batchDispatchRequestExecutor = new ThreadPoolExecutor(
+
DefaultMessageStore.this.getBrokerConfig().getBatchDispatchRequestThreadPoolNums(),
+
DefaultMessageStore.this.getBrokerConfig().getBatchDispatchRequestThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MICROSECONDS,
+ new LinkedBlockingQueue<>(4096),
+ new
ThreadFactoryImpl("BatchDispatchRequestServiceThread_"),
+ new ThreadPoolExecutor.AbortPolicy());
+ }
+
+ private void pollBatchDispatchRequest() {
+ try {
+ if (!batchDispatchRequestQueue.isEmpty()) {
+ BatchDispatchRequest task =
batchDispatchRequestQueue.peek();
+ batchDispatchRequestExecutor.execute(() -> {
+ try {
+ ByteBuffer tmpByteBuffer =
task.byteBuffer.duplicate();
+ tmpByteBuffer.position(task.position);
+ tmpByteBuffer.limit(task.position + task.size);
+ List<DispatchRequest> dispatchRequestList = new
ArrayList<>();
+ while (tmpByteBuffer.hasRemaining()) {
+ DispatchRequest dispatchRequest =
DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(tmpByteBuffer,
false, false, false);
+ if (dispatchRequest.isSuccess()) {
+ dispatchRequestList.add(dispatchRequest);
+ } else {
+ LOGGER.error("[BUG]read total count not
equals msg total size.");
+ }
+ }
+ dispatchRequestOrderlyQueue.put(task.id,
dispatchRequestList.toArray(new DispatchRequest[dispatchRequestList.size()]));
+ mappedPageHoldCount.getAndDecrement();
+ } catch (Exception e) {
+ LOGGER.error("There is an exception in task
execution.", e);
+ }
+ });
+ batchDispatchRequestQueue.poll();
+ }
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ @Override
+ public void run() {
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
started");
+
+ while (!this.isStopped()) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1);
+ pollBatchDispatchRequest();
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
end");
+ }
+
+ @Override
+ public String getServiceName() {
+ if
(DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {
+ return
DefaultMessageStore.this.getBrokerIdentity().getIdentifier() +
MainBatchDispatchRequestService.class.getSimpleName();
+ }
+ return MainBatchDispatchRequestService.class.getSimpleName();
+ }
+
+ }
+
+ class DispatchService extends ServiceThread {
+
+ private final List<DispatchRequest[]> dispatchRequestsList = new
ArrayList<>();
+
+ private void dispatch() {
+ dispatchRequestsList.clear();
+ dispatchRequestOrderlyQueue.get(dispatchRequestsList);
+ if (!dispatchRequestsList.isEmpty()) {
+ for (DispatchRequest[] dispatchRequests :
dispatchRequestsList) {
+ for (DispatchRequest dispatchRequest : dispatchRequests) {
+ DefaultMessageStore.this.doDispatch(dispatchRequest);
+ // wake up long-polling
+ if
(DefaultMessageStore.this.brokerConfig.isLongPollingEnable()
+ &&
DefaultMessageStore.this.messageArrivingListener != null) {
+
DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),
+ dispatchRequest.getQueueId(),
dispatchRequest.getConsumeQueueOffset() + 1,
+ dispatchRequest.getTagsCode(),
dispatchRequest.getStoreTimestamp(),
+ dispatchRequest.getBitMap(),
dispatchRequest.getPropertiesMap());
+
DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest);
+ }
+ if
(!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() &&
+
DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() ==
BrokerRole.SLAVE) {
+ DefaultMessageStore.this.storeStatsService
+
.getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1);
+ DefaultMessageStore.this.storeStatsService
+
.getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic())
+ .add(dispatchRequest.getMsgSize());
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
started");
+
+ while (!this.isStopped()) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1);
+ dispatch();
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
end");
+ }
+
+ @Override
+ public String getServiceName() {
+ if
(DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {
+ return
DefaultMessageStore.this.getBrokerIdentity().getIdentifier() +
DispatchService.class.getSimpleName();
+ }
+ return DispatchService.class.getSimpleName();
+ }
+ }
+
+ class ConcurrentReputMessageService extends ReputMessageService {
+
+ private static final int BATCH_SIZE = 1024 * 1024 * 4;
+
+ private long batchId = 0;
+
+ private MainBatchDispatchRequestService
mainBatchDispatchRequestService;
+
+ private DispatchService dispatchService;
+
+ public ConcurrentReputMessageService() {
+ super();
+ this.mainBatchDispatchRequestService = new
MainBatchDispatchRequestService();
+ this.dispatchService = new DispatchService();
+ }
+
+ public void createBatchDispatchRequest(ByteBuffer byteBuffer, int
position, int size) {
+ if (position < 0) {
+ return;
+ }
+ mappedPageHoldCount.getAndIncrement();
+ BatchDispatchRequest task = new BatchDispatchRequest(byteBuffer,
position, size, batchId++);
+ batchDispatchRequestQueue.offer(task);
+ }
+
+ @Override
+ public void start() {
+ super.start();
+ this.mainBatchDispatchRequestService.start();
+ this.dispatchService.start();
+ }
+
+ @Override
+ public void doReput() {
Review Comment:
The doReput method in ConcurrentReputMessageService is similar with the
method in class ReputMessageService. Is it possible to abstract some code
fragments to reuse the code?
##########
store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java:
##########
@@ -2743,6 +2835,282 @@ public String getServiceName() {
}
+ class MainBatchDispatchRequestService extends ServiceThread {
+
+ private final ExecutorService batchDispatchRequestExecutor;
+
+ public MainBatchDispatchRequestService() {
+ batchDispatchRequestExecutor = new ThreadPoolExecutor(
+
DefaultMessageStore.this.getBrokerConfig().getBatchDispatchRequestThreadPoolNums(),
+
DefaultMessageStore.this.getBrokerConfig().getBatchDispatchRequestThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MICROSECONDS,
+ new LinkedBlockingQueue<>(4096),
+ new
ThreadFactoryImpl("BatchDispatchRequestServiceThread_"),
+ new ThreadPoolExecutor.AbortPolicy());
+ }
+
+ private void pollBatchDispatchRequest() {
+ try {
+ if (!batchDispatchRequestQueue.isEmpty()) {
+ BatchDispatchRequest task =
batchDispatchRequestQueue.peek();
+ batchDispatchRequestExecutor.execute(() -> {
+ try {
+ ByteBuffer tmpByteBuffer =
task.byteBuffer.duplicate();
+ tmpByteBuffer.position(task.position);
+ tmpByteBuffer.limit(task.position + task.size);
+ List<DispatchRequest> dispatchRequestList = new
ArrayList<>();
+ while (tmpByteBuffer.hasRemaining()) {
+ DispatchRequest dispatchRequest =
DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(tmpByteBuffer,
false, false, false);
+ if (dispatchRequest.isSuccess()) {
+ dispatchRequestList.add(dispatchRequest);
+ } else {
+ LOGGER.error("[BUG]read total count not
equals msg total size.");
+ }
+ }
+ dispatchRequestOrderlyQueue.put(task.id,
dispatchRequestList.toArray(new DispatchRequest[dispatchRequestList.size()]));
+ mappedPageHoldCount.getAndDecrement();
+ } catch (Exception e) {
+ LOGGER.error("There is an exception in task
execution.", e);
+ }
+ });
+ batchDispatchRequestQueue.poll();
+ }
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ @Override
+ public void run() {
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
started");
+
+ while (!this.isStopped()) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1);
+ pollBatchDispatchRequest();
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
end");
+ }
+
+ @Override
+ public String getServiceName() {
+ if
(DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {
+ return
DefaultMessageStore.this.getBrokerIdentity().getIdentifier() +
MainBatchDispatchRequestService.class.getSimpleName();
+ }
+ return MainBatchDispatchRequestService.class.getSimpleName();
+ }
+
+ }
+
+ class DispatchService extends ServiceThread {
+
+ private final List<DispatchRequest[]> dispatchRequestsList = new
ArrayList<>();
+
+ private void dispatch() {
+ dispatchRequestsList.clear();
+ dispatchRequestOrderlyQueue.get(dispatchRequestsList);
+ if (!dispatchRequestsList.isEmpty()) {
+ for (DispatchRequest[] dispatchRequests :
dispatchRequestsList) {
+ for (DispatchRequest dispatchRequest : dispatchRequests) {
+ DefaultMessageStore.this.doDispatch(dispatchRequest);
+ // wake up long-polling
+ if
(DefaultMessageStore.this.brokerConfig.isLongPollingEnable()
+ &&
DefaultMessageStore.this.messageArrivingListener != null) {
+
DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),
+ dispatchRequest.getQueueId(),
dispatchRequest.getConsumeQueueOffset() + 1,
+ dispatchRequest.getTagsCode(),
dispatchRequest.getStoreTimestamp(),
+ dispatchRequest.getBitMap(),
dispatchRequest.getPropertiesMap());
+
DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest);
+ }
+ if
(!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() &&
+
DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() ==
BrokerRole.SLAVE) {
+ DefaultMessageStore.this.storeStatsService
+
.getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1);
+ DefaultMessageStore.this.storeStatsService
+
.getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic())
+ .add(dispatchRequest.getMsgSize());
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
started");
+
+ while (!this.isStopped()) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1);
+ dispatch();
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
end");
+ }
+
+ @Override
+ public String getServiceName() {
+ if
(DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {
+ return
DefaultMessageStore.this.getBrokerIdentity().getIdentifier() +
DispatchService.class.getSimpleName();
+ }
+ return DispatchService.class.getSimpleName();
+ }
+ }
+
+ class ConcurrentReputMessageService extends ReputMessageService {
+
+ private static final int BATCH_SIZE = 1024 * 1024 * 4;
+
+ private long batchId = 0;
+
+ private MainBatchDispatchRequestService
mainBatchDispatchRequestService;
+
+ private DispatchService dispatchService;
+
+ public ConcurrentReputMessageService() {
+ super();
+ this.mainBatchDispatchRequestService = new
MainBatchDispatchRequestService();
+ this.dispatchService = new DispatchService();
+ }
+
+ public void createBatchDispatchRequest(ByteBuffer byteBuffer, int
position, int size) {
+ if (position < 0) {
+ return;
+ }
+ mappedPageHoldCount.getAndIncrement();
+ BatchDispatchRequest task = new BatchDispatchRequest(byteBuffer,
position, size, batchId++);
+ batchDispatchRequestQueue.offer(task);
+ }
+
+ @Override
+ public void start() {
+ super.start();
+ this.mainBatchDispatchRequestService.start();
+ this.dispatchService.start();
+ }
+
+ @Override
+ public void doReput() {
+ if (this.reputFromOffset <
DefaultMessageStore.this.commitLog.getMinOffset()) {
+ LOGGER.warn("The reputFromOffset={} is smaller than
minPyOffset={}, this usually indicate that the dispatch behind too much and the
commitlog has expired.",
+ this.reputFromOffset,
DefaultMessageStore.this.commitLog.getMinOffset());
+ this.reputFromOffset =
DefaultMessageStore.this.commitLog.getMinOffset();
+ }
+ for (boolean doNext = true; this.isCommitLogAvailable() && doNext;
) {
+
+ SelectMappedBufferResult result =
DefaultMessageStore.this.commitLog.getData(reputFromOffset);
+
+ if (result == null) {
+ break;
+ }
+
+ int batchDispatchRequestStart = -1;
+ int batchDispatchRequestSize = -1;
+ try {
+ this.reputFromOffset = result.getStartOffset();
+
+ for (int readSize = 0; readSize < result.getSize() &&
reputFromOffset < DefaultMessageStore.this.getConfirmOffset() && doNext; ) {
+ ByteBuffer byteBuffer = result.getByteBuffer();
Review Comment:
It's better to have a method that is responsible for building
DispatchRequest including byteBuffer decode like checkMessageAndReturnSize.
##########
store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java:
##########
@@ -2743,6 +2852,269 @@ public String getServiceName() {
}
+ class MainBatchDispatchRequestService extends ServiceThread {
+
+ private final ExecutorService batchDispatchRequestExecutor;
+
+ public MainBatchDispatchRequestService() {
+ batchDispatchRequestExecutor = new ThreadPoolExecutor(
+
DefaultMessageStore.this.getBrokerConfig().getBatchDispatchRequestThreadPoolNums(),
+
DefaultMessageStore.this.getBrokerConfig().getBatchDispatchRequestThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MICROSECONDS,
+ new LinkedBlockingQueue<>(4096),
+ new
ThreadFactoryImpl("BatchDispatchRequestServiceThread_"),
+ new RejectedExecutionHandler() {
+ @Override
+ public void rejectedExecution(Runnable r,
ThreadPoolExecutor executor) {
+ try {
+ LOGGER.warn("Task {} is blocking put into the
workQueue", r);
+ executor.getQueue().put(r);
+ } catch (InterruptedException e) {
+ LOGGER.error("Task {} failed to put into the
workQueue", r);
+ }
+ }
+ });
+ }
+
+ private void pollBatchDispatchRequest() {
+ if (!batchDispatchRequestQueue.isEmpty()) {
+ BatchDispatchRequest task = batchDispatchRequestQueue.poll();
+ batchDispatchRequestExecutor.execute(() -> {
+ ByteBuffer tmpByteBuffer = task.byteBuffer.duplicate();
+ tmpByteBuffer.position(task.position);
+ tmpByteBuffer.limit(task.position + task.size);
+ List<DispatchRequest> dispatchRequestList = new
ArrayList<>();
+ while (tmpByteBuffer.hasRemaining()) {
+ DispatchRequest dispatchRequest =
DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(tmpByteBuffer,
false, false, false);
+ if (dispatchRequest.isSuccess()) {
+ dispatchRequestList.add(dispatchRequest);
+ } else {
+ LOGGER.error("[BUG]read total count not equals msg
total size.");
+ }
+ }
+ dispatchRequestOrderlyQueue.put(task.id,
dispatchRequestList.toArray(new DispatchRequest[dispatchRequestList.size()]));
+ mappedPageHoldCount.getAndDecrement();
+ });
+ }
+ }
+
+ @Override
+ public void run() {
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
started");
+
+ while (!this.isStopped()) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1);
+ pollBatchDispatchRequest();
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
end");
+ }
+
+ @Override
+ public String getServiceName() {
+ if
(DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {
+ return
DefaultMessageStore.this.getBrokerIdentity().getIdentifier() +
MainBatchDispatchRequestService.class.getSimpleName();
+ }
+ return MainBatchDispatchRequestService.class.getSimpleName();
+ }
+
+ }
+
+ class DispatchService extends ServiceThread {
+
+ private final List<DispatchRequest[]> dispatchRequestsList = new
ArrayList<>();
+
+ private void dispatch() {
+ dispatchRequestsList.clear();
+ dispatchRequestOrderlyQueue.get(dispatchRequestsList);
+ if (!dispatchRequestsList.isEmpty()) {
+ for (DispatchRequest[] dispatchRequests :
dispatchRequestsList) {
+ for (DispatchRequest dispatchRequest : dispatchRequests) {
+ DefaultMessageStore.this.doDispatch(dispatchRequest);
+ // wake up long-polling
+ if
(DefaultMessageStore.this.brokerConfig.isLongPollingEnable()
+ &&
DefaultMessageStore.this.messageArrivingListener != null) {
+
DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),
+ dispatchRequest.getQueueId(),
dispatchRequest.getConsumeQueueOffset() + 1,
+ dispatchRequest.getTagsCode(),
dispatchRequest.getStoreTimestamp(),
+ dispatchRequest.getBitMap(),
dispatchRequest.getPropertiesMap());
+
DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest);
+ }
+ if
(!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() &&
+
DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() ==
BrokerRole.SLAVE) {
+ DefaultMessageStore.this.storeStatsService
+
.getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1);
+ DefaultMessageStore.this.storeStatsService
+
.getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic())
+ .add(dispatchRequest.getMsgSize());
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
started");
+
+ while (!this.isStopped()) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1);
+ dispatch();
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
end");
+ }
+
+ @Override
+ public String getServiceName() {
+ if
(DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {
+ return
DefaultMessageStore.this.getBrokerIdentity().getIdentifier() +
DispatchService.class.getSimpleName();
+ }
+ return DispatchService.class.getSimpleName();
+ }
+ }
+
+ class ConcurrentReputMessageService extends ReputMessageService {
+
+ private static final int BATCH_SIZE = 1024 * 1024 * 4;
+
+ private int batchId = 0;
Review Comment:
It's recommended to use Long for batchId.
##########
store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java:
##########
@@ -2743,6 +2835,282 @@ public String getServiceName() {
}
+ class MainBatchDispatchRequestService extends ServiceThread {
+
+ private final ExecutorService batchDispatchRequestExecutor;
+
+ public MainBatchDispatchRequestService() {
+ batchDispatchRequestExecutor = new ThreadPoolExecutor(
+
DefaultMessageStore.this.getBrokerConfig().getBatchDispatchRequestThreadPoolNums(),
+
DefaultMessageStore.this.getBrokerConfig().getBatchDispatchRequestThreadPoolNums(),
+ 1000 * 60,
+ TimeUnit.MICROSECONDS,
+ new LinkedBlockingQueue<>(4096),
+ new
ThreadFactoryImpl("BatchDispatchRequestServiceThread_"),
+ new ThreadPoolExecutor.AbortPolicy());
+ }
+
+ private void pollBatchDispatchRequest() {
+ try {
+ if (!batchDispatchRequestQueue.isEmpty()) {
+ BatchDispatchRequest task =
batchDispatchRequestQueue.peek();
+ batchDispatchRequestExecutor.execute(() -> {
+ try {
+ ByteBuffer tmpByteBuffer =
task.byteBuffer.duplicate();
+ tmpByteBuffer.position(task.position);
+ tmpByteBuffer.limit(task.position + task.size);
+ List<DispatchRequest> dispatchRequestList = new
ArrayList<>();
+ while (tmpByteBuffer.hasRemaining()) {
+ DispatchRequest dispatchRequest =
DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(tmpByteBuffer,
false, false, false);
+ if (dispatchRequest.isSuccess()) {
+ dispatchRequestList.add(dispatchRequest);
+ } else {
+ LOGGER.error("[BUG]read total count not
equals msg total size.");
+ }
+ }
+ dispatchRequestOrderlyQueue.put(task.id,
dispatchRequestList.toArray(new DispatchRequest[dispatchRequestList.size()]));
+ mappedPageHoldCount.getAndDecrement();
+ } catch (Exception e) {
+ LOGGER.error("There is an exception in task
execution.", e);
+ }
+ });
+ batchDispatchRequestQueue.poll();
+ }
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ @Override
+ public void run() {
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
started");
+
+ while (!this.isStopped()) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1);
+ pollBatchDispatchRequest();
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
end");
+ }
+
+ @Override
+ public String getServiceName() {
+ if
(DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {
+ return
DefaultMessageStore.this.getBrokerIdentity().getIdentifier() +
MainBatchDispatchRequestService.class.getSimpleName();
+ }
+ return MainBatchDispatchRequestService.class.getSimpleName();
+ }
+
+ }
+
+ class DispatchService extends ServiceThread {
+
+ private final List<DispatchRequest[]> dispatchRequestsList = new
ArrayList<>();
+
+ private void dispatch() {
+ dispatchRequestsList.clear();
+ dispatchRequestOrderlyQueue.get(dispatchRequestsList);
+ if (!dispatchRequestsList.isEmpty()) {
+ for (DispatchRequest[] dispatchRequests :
dispatchRequestsList) {
+ for (DispatchRequest dispatchRequest : dispatchRequests) {
+ DefaultMessageStore.this.doDispatch(dispatchRequest);
+ // wake up long-polling
+ if
(DefaultMessageStore.this.brokerConfig.isLongPollingEnable()
+ &&
DefaultMessageStore.this.messageArrivingListener != null) {
+
DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),
+ dispatchRequest.getQueueId(),
dispatchRequest.getConsumeQueueOffset() + 1,
+ dispatchRequest.getTagsCode(),
dispatchRequest.getStoreTimestamp(),
+ dispatchRequest.getBitMap(),
dispatchRequest.getPropertiesMap());
+
DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest);
+ }
+ if
(!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() &&
+
DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() ==
BrokerRole.SLAVE) {
+ DefaultMessageStore.this.storeStatsService
+
.getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1);
+ DefaultMessageStore.this.storeStatsService
+
.getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic())
+ .add(dispatchRequest.getMsgSize());
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
started");
+
+ while (!this.isStopped()) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(1);
+ dispatch();
+ } catch (Exception e) {
+ DefaultMessageStore.LOGGER.warn(this.getServiceName() + "
service has exception. ", e);
+ }
+ }
+
+ DefaultMessageStore.LOGGER.info(this.getServiceName() + " service
end");
+ }
+
+ @Override
+ public String getServiceName() {
+ if
(DefaultMessageStore.this.getBrokerConfig().isInBrokerContainer()) {
+ return
DefaultMessageStore.this.getBrokerIdentity().getIdentifier() +
DispatchService.class.getSimpleName();
+ }
+ return DispatchService.class.getSimpleName();
+ }
+ }
+
+ class ConcurrentReputMessageService extends ReputMessageService {
+
+ private static final int BATCH_SIZE = 1024 * 1024 * 4;
+
+ private long batchId = 0;
+
+ private MainBatchDispatchRequestService
mainBatchDispatchRequestService;
+
+ private DispatchService dispatchService;
+
+ public ConcurrentReputMessageService() {
+ super();
+ this.mainBatchDispatchRequestService = new
MainBatchDispatchRequestService();
+ this.dispatchService = new DispatchService();
+ }
+
+ public void createBatchDispatchRequest(ByteBuffer byteBuffer, int
position, int size) {
+ if (position < 0) {
+ return;
+ }
+ mappedPageHoldCount.getAndIncrement();
+ BatchDispatchRequest task = new BatchDispatchRequest(byteBuffer,
position, size, batchId++);
+ batchDispatchRequestQueue.offer(task);
+ }
+
+ @Override
+ public void start() {
+ super.start();
+ this.mainBatchDispatchRequestService.start();
+ this.dispatchService.start();
+ }
+
+ @Override
+ public void doReput() {
+ if (this.reputFromOffset <
DefaultMessageStore.this.commitLog.getMinOffset()) {
+ LOGGER.warn("The reputFromOffset={} is smaller than
minPyOffset={}, this usually indicate that the dispatch behind too much and the
commitlog has expired.",
+ this.reputFromOffset,
DefaultMessageStore.this.commitLog.getMinOffset());
+ this.reputFromOffset =
DefaultMessageStore.this.commitLog.getMinOffset();
+ }
+ for (boolean doNext = true; this.isCommitLogAvailable() && doNext;
) {
+
+ SelectMappedBufferResult result =
DefaultMessageStore.this.commitLog.getData(reputFromOffset);
+
+ if (result == null) {
+ break;
+ }
+
+ int batchDispatchRequestStart = -1;
+ int batchDispatchRequestSize = -1;
+ try {
+ this.reputFromOffset = result.getStartOffset();
+
+ for (int readSize = 0; readSize < result.getSize() &&
reputFromOffset < DefaultMessageStore.this.getConfirmOffset() && doNext; ) {
+ ByteBuffer byteBuffer = result.getByteBuffer();
+
+ byteBuffer.mark();
+
+ int totalSize = byteBuffer.getInt();
+ if (reputFromOffset + totalSize >
DefaultMessageStore.this.getConfirmOffset()) {
+ doNext = false;
+ break;
+ }
+
+ int magicCode = byteBuffer.getInt();
+ switch (magicCode) {
+ case CommitLog.MESSAGE_MAGIC_CODE:
Review Comment:
MESSAGE_MAGIC_CODE_V2 should be put into consideration.
See
https://github.com/apache/rocketmq/blob/aa8ca48d478e58309f9a6b026e6fea7f4772c6e9/store/src/main/java/org/apache/rocketmq/store/CommitLog.java#L386-L391
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
