diff --git a/broker/pom.xml b/broker/pom.xml index 0137b556789510354941a1e93f2ac67967fd8648..64ae7d995519e235b5cd7ee44ad96ce037febd2a 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -66,6 +66,10 @@ org.slf4j slf4j-api + + com.googlecode.concurrentlinkedhashmap + concurrentlinkedhashmap-lru + diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 194f2850fa57274d6a620c72c403ba1c5f1868dd..e83bea239753a5314e48876b03edbc97e838d155 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -47,19 +47,25 @@ import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.broker.filtersrv.FilterServerManager; import org.apache.rocketmq.broker.latency.BrokerFastFailure; import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; +import org.apache.rocketmq.broker.loadbalance.AssignmentManager; import org.apache.rocketmq.broker.longpolling.NotifyMessageArrivingListener; import org.apache.rocketmq.broker.longpolling.PullRequestHoldService; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; +import org.apache.rocketmq.broker.offset.ConsumerOrderInfoManager; import org.apache.rocketmq.broker.out.BrokerOuterAPI; import org.apache.rocketmq.broker.plugin.MessageStoreFactory; import org.apache.rocketmq.broker.plugin.MessageStorePluginContext; +import org.apache.rocketmq.broker.processor.AckMessageProcessor; import org.apache.rocketmq.broker.processor.AdminBrokerProcessor; +import org.apache.rocketmq.broker.processor.ChangeInvisibleTimeProcessor; import org.apache.rocketmq.broker.processor.ClientManageProcessor; import org.apache.rocketmq.broker.processor.ConsumerManageProcessor; import org.apache.rocketmq.broker.processor.EndTransactionProcessor; +import org.apache.rocketmq.broker.processor.PopMessageProcessor; import org.apache.rocketmq.broker.processor.PullMessageProcessor; +import org.apache.rocketmq.broker.processor.QueryAssignmentProcessor; import org.apache.rocketmq.broker.processor.QueryMessageProcessor; import org.apache.rocketmq.broker.processor.ReplyMessageProcessor; import org.apache.rocketmq.broker.processor.SendMessageProcessor; @@ -118,9 +124,18 @@ public class BrokerController { private final ConsumerOffsetManager consumerOffsetManager; private final ConsumerManager consumerManager; private final ConsumerFilterManager consumerFilterManager; + private final ConsumerOrderInfoManager consumerOrderInfoManager; private final ProducerManager producerManager; + private final AssignmentManager assignmentManager; private final ClientHousekeepingService clientHousekeepingService; + private final PullMessageProcessor pullMessageProcessor; + private final PopMessageProcessor popMessageProcessor; + private final AckMessageProcessor ackMessageProcessor; + private final ChangeInvisibleTimeProcessor changeInvisibleTimeProcessor; + private final QueryAssignmentProcessor queryAssignmentProcessor; + private final ClientManageProcessor clientManageProcessor; + private final SendMessageProcessor sendMessageProcessor; private final PullRequestHoldService pullRequestHoldService; private final MessageArrivingListener messageArrivingListener; private final Broker2Client broker2Client; @@ -132,6 +147,7 @@ public class BrokerController { "BrokerControllerScheduledThread")); private final SlaveSynchronize slaveSynchronize; private final BlockingQueue sendThreadPoolQueue; + private final BlockingQueue ackThreadPoolQueue; private final BlockingQueue pullThreadPoolQueue; private final BlockingQueue replyThreadPoolQueue; private final BlockingQueue queryThreadPoolQueue; @@ -149,12 +165,14 @@ public class BrokerController { private TopicConfigManager topicConfigManager; private ExecutorService sendMessageExecutor; private ExecutorService pullMessageExecutor; + private ExecutorService ackMessageExecutor; private ExecutorService replyMessageExecutor; private ExecutorService queryMessageExecutor; private ExecutorService adminBrokerExecutor; private ExecutorService clientManageExecutor; private ExecutorService heartbeatExecutor; private ExecutorService consumerManageExecutor; + private ExecutorService loadBalanceExecutor; private ExecutorService endTransactionExecutor; private boolean updateMasterHAServerAddrPeriodically = false; private BrokerStats brokerStats; @@ -167,6 +185,7 @@ public class BrokerController { private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener; private Future slaveSyncFuture; private Map accessValidatorMap = new HashMap(); + private long shouldStartTime; public BrokerController( final BrokerConfig brokerConfig, @@ -182,10 +201,16 @@ public class BrokerController { this.topicConfigManager = new TopicConfigManager(this); this.pullMessageProcessor = new PullMessageProcessor(this); this.pullRequestHoldService = new PullRequestHoldService(this); - this.messageArrivingListener = new NotifyMessageArrivingListener(this.pullRequestHoldService); + this.popMessageProcessor = new PopMessageProcessor(this); + this.ackMessageProcessor = new AckMessageProcessor(this); + this.changeInvisibleTimeProcessor = new ChangeInvisibleTimeProcessor(this); + this.sendMessageProcessor = new SendMessageProcessor(this); + this.messageArrivingListener = new NotifyMessageArrivingListener(this.pullRequestHoldService, + this.popMessageProcessor); this.consumerIdsChangeListener = new DefaultConsumerIdsChangeListener(this); this.consumerManager = new ConsumerManager(this.consumerIdsChangeListener); this.consumerFilterManager = new ConsumerFilterManager(this); + this.consumerOrderInfoManager = new ConsumerOrderInfoManager(this); this.producerManager = new ProducerManager(); this.clientHousekeepingService = new ClientHousekeepingService(this); this.broker2Client = new Broker2Client(this); @@ -193,10 +218,14 @@ public class BrokerController { this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig); this.filterServerManager = new FilterServerManager(this); + this.assignmentManager = new AssignmentManager(this); + this.queryAssignmentProcessor = new QueryAssignmentProcessor(this); + this.clientManageProcessor = new ClientManageProcessor(this); this.slaveSynchronize = new SlaveSynchronize(this); this.sendThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getSendThreadPoolQueueCapacity()); this.pullThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getPullThreadPoolQueueCapacity()); + this.ackThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getAckThreadPoolQueueCapacity()); this.replyThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getReplyThreadPoolQueueCapacity()); this.queryThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getQueryThreadPoolQueueCapacity()); this.clientManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getClientManagerThreadPoolQueueCapacity()); @@ -215,6 +244,14 @@ public class BrokerController { ); } + public ConsumerIdsChangeListener getConsumerIdsChangeListener() { + return consumerIdsChangeListener; + } + + public ClientManageProcessor getClientManageProcessor() { + return clientManageProcessor; + } + public BrokerConfig getBrokerConfig() { return brokerConfig; } @@ -281,6 +318,15 @@ public class BrokerController { this.pullThreadPoolQueue, new ThreadFactoryImpl("PullMessageThread_")); + this.ackMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.brokerConfig.getAckMessageThreadPoolNums(), + this.brokerConfig.getAckMessageThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.ackThreadPoolQueue, + new ThreadFactoryImpl("AckMessageThread_")); + + this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor( this.brokerConfig.getProcessReplyMessageThreadPoolNums(), this.brokerConfig.getProcessReplyMessageThreadPoolNums(), @@ -400,6 +446,10 @@ public class BrokerController { } }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); + this.loadBalanceExecutor = + Executors.newFixedThreadPool(this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(), new ThreadFactoryImpl( + "LoadBalanceProcessorThread_")); + if (this.brokerConfig.getNamesrvAddr() != null) { this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr()); log.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr()); @@ -547,23 +597,38 @@ public class BrokerController { /** * SendMessageProcessor */ - SendMessageProcessor sendProcessor = new SendMessageProcessor(this); - sendProcessor.registerSendMessageHook(sendMessageHookList); - sendProcessor.registerConsumeMessageHook(consumeMessageHookList); - - this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendProcessor, this.sendMessageExecutor); - this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendProcessor, this.sendMessageExecutor); - this.remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendProcessor, this.sendMessageExecutor); - this.remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendProcessor, this.sendMessageExecutor); - this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendProcessor, this.sendMessageExecutor); - this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendProcessor, this.sendMessageExecutor); - this.fastRemotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendProcessor, this.sendMessageExecutor); - this.fastRemotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendProcessor, this.sendMessageExecutor); + + sendMessageProcessor.registerSendMessageHook(sendMessageHookList); + sendMessageProcessor.registerConsumeMessageHook(consumeMessageHookList); + + this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor); /** * PullMessageProcessor */ this.remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, this.pullMessageExecutor); this.pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList); + /** + * PopMessageProcessor + */ + this.remotingServer.registerProcessor(RequestCode.POP_MESSAGE, this.popMessageProcessor, this.pullMessageExecutor); + + /** + * AckMessageProcessor + */ + this.remotingServer.registerProcessor(RequestCode.ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.ACK_MESSAGE, this.ackMessageProcessor, this.ackMessageExecutor); + /** + * ChangeInvisibleTimeProcessor + */ + this.remotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, this.changeInvisibleTimeProcessor, this.ackMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, this.changeInvisibleTimeProcessor, this.ackMessageExecutor); /** * ReplyMessageProcessor @@ -589,14 +654,13 @@ public class BrokerController { /** * ClientManageProcessor */ - ClientManageProcessor clientProcessor = new ClientManageProcessor(this); - this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.heartbeatExecutor); - this.remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor); - this.remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor); + this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManageProcessor, this.heartbeatExecutor); + this.remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManageProcessor, this.clientManageExecutor); + this.remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManageProcessor, this.clientManageExecutor); - this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.heartbeatExecutor); - this.fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor); - this.fastRemotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientManageProcessor, this.heartbeatExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientManageProcessor, this.clientManageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientManageProcessor, this.clientManageExecutor); /** * ConsumerManageProcessor @@ -610,6 +674,14 @@ public class BrokerController { this.fastRemotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor); this.fastRemotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor); + /** + * QueryAssignmentProcessor + */ + this.remotingServer.registerProcessor(RequestCode.QUERY_ASSIGNMENT, queryAssignmentProcessor, loadBalanceExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.QUERY_ASSIGNMENT, queryAssignmentProcessor, loadBalanceExecutor); + this.remotingServer.registerProcessor(RequestCode.SET_MESSAGE_REQUEST_MODE, queryAssignmentProcessor, loadBalanceExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.SET_MESSAGE_REQUEST_MODE, queryAssignmentProcessor, loadBalanceExecutor); + /** * EndTransactionProcessor */ @@ -713,6 +785,10 @@ public class BrokerController { return consumerFilterManager; } + public ConsumerOrderInfoManager getConsumerOrderInfoManager() { + return consumerOrderInfoManager; + } + public ConsumerOffsetManager getConsumerOffsetManager() { return consumerOffsetManager; } @@ -741,6 +817,10 @@ public class BrokerController { return subscriptionGroupManager; } + public PopMessageProcessor getPopMessageProcessor() { + return popMessageProcessor; + } + public void shutdown() { if (this.brokerStatsManager != null) { this.brokerStatsManager.shutdown(); @@ -824,6 +904,11 @@ public class BrokerController { this.consumerManageExecutor.shutdown(); } + { + this.popMessageProcessor.getPopBufferMergeService().shutdown(); + this.ackMessageProcessor.shutdownPopReviveService(); + } + if (this.fileWatchService != null) { this.fileWatchService.shutdown(); } @@ -849,6 +934,8 @@ public class BrokerController { } public void start() throws Exception { + this.shouldStartTime = System.currentTimeMillis(); + if (this.messageStore != null) { this.messageStore.start(); } @@ -857,6 +944,17 @@ public class BrokerController { this.remotingServer.start(); } + { + this.popMessageProcessor.getPopLongPollingService().start(); + this.popMessageProcessor.getPopBufferMergeService().start(); + this.popMessageProcessor.getQueueLockManager().start(); + this.ackMessageProcessor.startPopReviveService(); + } + + { + assignmentManager.start(); + } + if (this.fastRemotingServer != null) { this.fastRemotingServer.start(); } @@ -1243,4 +1341,20 @@ public class BrokerController { public ExecutorService getSendMessageExecutor() { return sendMessageExecutor; } + + public long getShouldStartTime() { + return shouldStartTime; + } + + public AssignmentManager getAssignmentManager() { + return assignmentManager; + } + + public SendMessageProcessor getSendMessageProcessor() { + return sendMessageProcessor; + } + + public QueryAssignmentProcessor getQueryAssignmentProcessor() { + return queryAssignmentProcessor; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java index 42c8da9f3fba7ab563a898b6191dc0b4246b9df2..43a9946f7106c1f2f4506155c70b3a13e115cde4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java @@ -39,6 +39,10 @@ public class BrokerPathConfigHelper { return rootDir + File.separator + "config" + File.separator + "consumerOffset.json"; } + public static String getConsumerOrderInfoPath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "consumerOrderInfo.json"; + } + public static String getSubscriptionGroupPath(final String rootDir) { return rootDir + File.separator + "config" + File.separator + "subscriptionGroup.json"; } @@ -46,4 +50,8 @@ public class BrokerPathConfigHelper { public static String getConsumerFilterPath(final String rootDir) { return rootDir + File.separator + "config" + File.separator + "consumerFilter.json"; } + + public static String getMessageRequestModePath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "messageRequestMode.json"; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java b/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java new file mode 100644 index 0000000000000000000000000000000000000000..f4c30d5d648141fa6b00a16ca16d9d1d0486f9fc --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.loadbalance; + +import com.google.common.collect.Lists; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.out.BrokerOuterAPI; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + + +public class AssignmentManager { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + + private transient BrokerController brokerController; + + private final static long LOCK_TIMEOUT_MILLIS = 3000; + + private final Lock lockNamesrv = new ReentrantLock(); + + private final BrokerOuterAPI mQClientAPIImpl; + + private final ConcurrentHashMap> topicSubscribeInfoTable = new ConcurrentHashMap>(); + + private ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("LoadBalanceManagerScheduledThread")); + + private static final List IGNORE_ROUTE_TOPICS = Lists.newArrayList( + TopicValidator.SYSTEM_TOPIC_PREFIX, + MixAll.CID_RMQ_SYS_PREFIX, + MixAll.DEFAULT_CONSUMER_GROUP, + MixAll.TOOLS_CONSUMER_GROUP, + MixAll.FILTERSRV_CONSUMER_GROUP, + MixAll.MONITOR_CONSUMER_GROUP, + MixAll.ONS_HTTP_PROXY_GROUP, + MixAll.CID_ONSAPI_PERMISSION_GROUP, + MixAll.CID_ONSAPI_OWNER_GROUP, + MixAll.CID_ONSAPI_PULL_GROUP + ); + + private final List ignoreRouteTopics = Lists.newArrayList(IGNORE_ROUTE_TOPICS); + + public AssignmentManager(BrokerController brokerController) { + this.brokerController = brokerController; + this.mQClientAPIImpl = brokerController.getBrokerOuterAPI(); + ignoreRouteTopics.add(brokerController.getBrokerConfig().getBrokerClusterName()); + ignoreRouteTopics.add(brokerController.getBrokerConfig().getBrokerName()); + } + + public void start() { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + updateTopicRouteInfoFromNameServer(); + } catch (Exception e) { + log.error("ScheduledTask: failed to pull TopicRouteData from NameServer", e); + } + } + }, 13000, this.brokerController.getBrokerConfig().getLoadBalancePollNameServerInterval(), TimeUnit.MILLISECONDS); + } + + + public void updateTopicRouteInfoFromNameServer() { + Set topicList = new HashSet<>(brokerController.getTopicConfigManager().getTopicConfigTable().keySet()); + + LOOP: + for (String topic : topicList) { + for (String keyword : ignoreRouteTopics) { + if (topic.contains(keyword)) { + continue LOOP; + } + } + + this.updateTopicRouteInfoFromNameServer(topic); + } + } + + public boolean updateTopicRouteInfoFromNameServer(final String topic) { + try { + TopicRouteData topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, 1000 * 3); + if (topicRouteData != null) { + Set newSubscribeInfo = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData); + Set oldSubscribeInfo = topicSubscribeInfoTable.get(topic); + boolean changed = !newSubscribeInfo.equals(oldSubscribeInfo); + + if (changed) { + log.info("the topic[{}] subscribe message queue changed, old[{}] ,new[{}]", topic, oldSubscribeInfo, newSubscribeInfo); + topicSubscribeInfoTable.put(topic, newSubscribeInfo); + return true; + } + } else { + log.warn("updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}", topic); + } + } catch (Exception e) { + if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + log.warn("updateTopicRouteInfoFromNameServer Exception", e); + if (e instanceof MQBrokerException && ResponseCode.TOPIC_NOT_EXIST == ((MQBrokerException) e).getResponseCode()) { + // clean no used topic + cleanNoneRouteTopic(topic); + } + } + } + return false; + } + + private void cleanNoneRouteTopic(String topic) { + // clean no used topic + topicSubscribeInfoTable.remove(topic); + } + + + public Set getTopicSubscribeInfo(String topic) { + return topicSubscribeInfoTable.get(topic); + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/MessageRequestModeManager.java b/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/MessageRequestModeManager.java new file mode 100644 index 0000000000000000000000000000000000000000..2b21d3a6dfa81985c8ca1a2d1f435323a325891f --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/MessageRequestModeManager.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.loadbalance; + +import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.BrokerPathConfigHelper; +import org.apache.rocketmq.common.ConfigManager; +import org.apache.rocketmq.common.protocol.body.SetMessageRequestModeRequestBody; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + + +public class MessageRequestModeManager extends ConfigManager { + + private BrokerController brokerController; + + private ConcurrentHashMap> + messageRequestModeMap = new ConcurrentHashMap>(); + + public MessageRequestModeManager() { + // empty construct for decode + } + + public MessageRequestModeManager(BrokerController brokerController) { + this.brokerController = brokerController; + } + + public void setMessageRequestMode(String topic, String consumerGroup, SetMessageRequestModeRequestBody requestBody) { + ConcurrentHashMap consumerGroup2ModeMap = messageRequestModeMap.get(topic); + if (consumerGroup2ModeMap == null) { + consumerGroup2ModeMap = new ConcurrentHashMap(); + ConcurrentHashMap pre = + messageRequestModeMap.putIfAbsent(topic, consumerGroup2ModeMap); + if (pre != null) { + consumerGroup2ModeMap = pre; + } + } + consumerGroup2ModeMap.put(consumerGroup, requestBody); + } + + public SetMessageRequestModeRequestBody getMessageRequestMode(String topic, String consumerGroup) { + ConcurrentHashMap consumerGroup2ModeMap = messageRequestModeMap.get(topic); + if (consumerGroup2ModeMap != null) { + return consumerGroup2ModeMap.get(consumerGroup); + } + + return null; + } + + public ConcurrentHashMap> getMessageRequestModeMap() { + return this.messageRequestModeMap; + } + + public void setMessageRequestModeMap(ConcurrentHashMap> messageRequestModeMap) { + this.messageRequestModeMap = messageRequestModeMap; + } + + + @Override + public String encode() { + return this.encode(false); + } + + + @Override + public String configFilePath() { + return BrokerPathConfigHelper.getMessageRequestModePath(this.brokerController.getMessageStoreConfig().getStorePathRootDir()); + } + + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + MessageRequestModeManager obj = RemotingSerializable.fromJson(jsonString, MessageRequestModeManager.class); + if (obj != null) { + this.messageRequestModeMap = obj.messageRequestModeMap; + } + } + } + + @Override + public String encode(boolean prettyFormat) { + return RemotingSerializable.toJson(this, prettyFormat); + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java index ff0901126f2771b58136208d6012722a9b20f037..780346210c2cb3676f4bde960d356d8264321ae0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java @@ -17,15 +17,17 @@ package org.apache.rocketmq.broker.longpolling; -import org.apache.rocketmq.store.MessageArrivingListener; - import java.util.Map; +import org.apache.rocketmq.broker.processor.PopMessageProcessor; +import org.apache.rocketmq.store.MessageArrivingListener; public class NotifyMessageArrivingListener implements MessageArrivingListener { private final PullRequestHoldService pullRequestHoldService; + private final PopMessageProcessor popMessageProcessor; - public NotifyMessageArrivingListener(final PullRequestHoldService pullRequestHoldService) { + public NotifyMessageArrivingListener(final PullRequestHoldService pullRequestHoldService, final PopMessageProcessor popMessageProcessor) { this.pullRequestHoldService = pullRequestHoldService; + this.popMessageProcessor = popMessageProcessor; } @Override @@ -33,5 +35,6 @@ public class NotifyMessageArrivingListener implements MessageArrivingListener { long msgStoreTime, byte[] filterBitMap, Map properties) { this.pullRequestHoldService.notifyMessageArriving(topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties); + this.popMessageProcessor.notifyMessageArriving(topic, queueId); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java new file mode 100644 index 0000000000000000000000000000000000000000..e8e45674d23c12cc5ac998ac2255e74bb0e05145 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.longpolling; + +import io.netty.channel.Channel; +import java.util.Comparator; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class PopRequest { + private static final AtomicLong COUNTER = new AtomicLong(Long.MIN_VALUE); + + private RemotingCommand remotingCommand; + private Channel channel; + private long expired; + private AtomicBoolean complete = new AtomicBoolean(false); + private final long op = COUNTER.getAndIncrement(); + + public PopRequest(RemotingCommand remotingCommand, Channel channel, long expired) { + this.channel = channel; + this.remotingCommand = remotingCommand; + this.expired = expired; + } + + public Channel getChannel() { + return channel; + } + + public RemotingCommand getRemotingCommand() { + return remotingCommand; + } + + public boolean isTimeout() { + return System.currentTimeMillis() > (expired - 50); + } + + public boolean complete() { + return complete.compareAndSet(false, true); + } + + public long getExpired() { + return expired; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("PopRequest{"); + sb.append("cmd=").append(remotingCommand); + sb.append(", channel=").append(channel); + sb.append(", expired=").append(expired); + sb.append(", complete=").append(complete); + sb.append(", op=").append(op); + sb.append('}'); + return sb.toString(); + } + + public static final Comparator COMPARATOR = new Comparator() { + @Override + public int compare(PopRequest o1, PopRequest o2) { + int ret = (int) (o1.getExpired() - o2.getExpired()); + + if (ret != 0) { + return ret; + } + ret = (int) (o1.op - o2.op); + if (ret != 0) { + return ret; + } + return -1; + } + }; +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManager.java new file mode 100644 index 0000000000000000000000000000000000000000..68c767fd405dcf0ad74e6d8baae85ce42a4e1f28 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManager.java @@ -0,0 +1,426 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.offset; + +import com.alibaba.fastjson.annotation.JSONField; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.BrokerPathConfigHelper; +import org.apache.rocketmq.common.ConfigManager; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class ConsumerOrderInfoManager extends ConfigManager { + + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final String TOPIC_GROUP_SEPARATOR = "@"; + private static final long CLEAN_SPAN_FROM_LAST = 24 * 3600 * 1000; + + private ConcurrentHashMap> table = + new ConcurrentHashMap<>(128); + + private transient BrokerController brokerController; + + public ConsumerOrderInfoManager() { + } + + public ConsumerOrderInfoManager(BrokerController brokerController) { + this.brokerController = brokerController; + } + + public ConcurrentHashMap> getTable() { + return table; + } + + public void setTable(ConcurrentHashMap> table) { + this.table = table; + } + + /** + * not thread safe. + * + * @param topic + * @param group + * @param queueId + * @param msgOffsetList + */ + public int update(String topic, String group, int queueId, List msgOffsetList) { + String key = topic + TOPIC_GROUP_SEPARATOR + group; + ConcurrentHashMap qs = table.get(key); + if (qs == null) { + qs = new ConcurrentHashMap<>(16); + ConcurrentHashMap old = table.putIfAbsent(key, qs); + if (old != null) { + qs = old; + } + } + + OrderInfo orderInfo = qs.get(queueId); + + // start is same. + List simple = OrderInfo.simpleO(msgOffsetList); + if (orderInfo != null && simple.get(0).equals(orderInfo.getOffsetList().get(0))) { + if (simple.equals(orderInfo.getOffsetList())) { + orderInfo.setConsumedCount(orderInfo.getConsumedCount() + 1); + } else { + // reset, because msgs are changed. + orderInfo.setConsumedCount(0); + } + orderInfo.setLastConsumeTimestamp(System.currentTimeMillis()); + orderInfo.setOffsetList(simple); + orderInfo.setCommitOffsetBit(0); + } else { + orderInfo = new OrderInfo(); + orderInfo.setOffsetList(simple); + orderInfo.setLastConsumeTimestamp(System.currentTimeMillis()); + orderInfo.setConsumedCount(0); + orderInfo.setCommitOffsetBit(0); + + qs.put(queueId, orderInfo); + } + + return orderInfo.getConsumedCount(); + } + + public boolean checkBlock(String topic, String group, int queueId, long invisibleTime) { + String key = topic + TOPIC_GROUP_SEPARATOR + group; + ConcurrentHashMap qs = table.get(key); + if (qs == null) { + qs = new ConcurrentHashMap<>(16); + ConcurrentHashMap old = table.putIfAbsent(key, qs); + if (old != null) { + qs = old; + } + } + + OrderInfo orderInfo = qs.get(queueId); + + if (orderInfo == null) { + return false; + } + + boolean isBlock = System.currentTimeMillis() - orderInfo.getLastConsumeTimestamp() < invisibleTime; + + return isBlock && !orderInfo.isDone(); + } + + /** + * @param topic + * @param group + * @param queueId + * @param offset + * @return -1 : illegal, -2 : no need commit, >= 0 : commit + */ + public long commitAndNext(String topic, String group, int queueId, long offset) { + String key = topic + TOPIC_GROUP_SEPARATOR + group; + ConcurrentHashMap qs = table.get(key); + + if (qs == null) { + return offset + 1; + } + OrderInfo orderInfo = qs.get(queueId); + if (orderInfo == null) { + log.warn("OrderInfo is null, {}, {}, {}", key, offset, orderInfo); + return offset + 1; + } + + List offsetList = orderInfo.getOffsetList(); + if (offsetList == null || offsetList.isEmpty()) { + log.warn("OrderInfo is empty, {}, {}, {}", key, offset, orderInfo); + return -1; + } + Long first = offsetList.get(0); + int i = 0, size = offsetList.size(); + for (; i < size; i++) { + long temp; + if (i == 0) { + temp = first; + } else { + temp = first + offsetList.get(i); + } + if (offset == temp) { + break; + } + } + // not found + if (i >= size) { + log.warn("OrderInfo not found commit offset, {}, {}, {}", key, offset, orderInfo); + return -1; + } + //set bit + orderInfo.setCommitOffsetBit(orderInfo.getCommitOffsetBit() | (1L << i)); + if (orderInfo.isDone()) { + if (size == 1) { + return offsetList.get(0) + 1; + } else { + return offsetList.get(size - 1) + first + 1; + } + } + return -2; + } + + public OrderInfo get(String topic, String group, int queueId) { + String key = topic + TOPIC_GROUP_SEPARATOR + group; + ConcurrentHashMap qs = table.get(key); + + if (qs == null) { + return null; + } + + return qs.get(queueId); + } + + public int getConsumeCount(String topic, String group, int queueId) { + OrderInfo orderInfo = get(topic, group, queueId); + return orderInfo == null ? 0 : orderInfo.getConsumedCount(); + } + + private void autoClean() { + if (brokerController == null) { + return; + } + Iterator>> iterator = + this.table.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> entry = + iterator.next(); + String topicAtGroup = entry.getKey(); + ConcurrentHashMap qs = entry.getValue(); + String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR); + if (arrays.length != 2) { + continue; + } + String topic = arrays[0]; + String group = arrays[1]; + + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (topicConfig == null) { + iterator.remove(); + log.info("Topic not exist, Clean order info, {}:{}", topicAtGroup, qs); + continue; + } + + if (this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().get(group) == null) { + iterator.remove(); + log.info("Group not exist, Clean order info, {}:{}", topicAtGroup, qs); + continue; + } + + if (qs.isEmpty()) { + iterator.remove(); + log.info("Order table is empty, Clean order info, {}:{}", topicAtGroup, qs); + continue; + } + + Iterator> qsIterator = qs.entrySet().iterator(); + while (qsIterator.hasNext()) { + Map.Entry qsEntry = qsIterator.next(); + + if (qsEntry.getKey() >= topicConfig.getReadQueueNums()) { + qsIterator.remove(); + log.info("Queue not exist, Clean order info, {}:{}, {}", topicAtGroup, entry.getValue(), topicConfig); + continue; + } + + if (System.currentTimeMillis() - qsEntry.getValue().getLastConsumeTimestamp() > CLEAN_SPAN_FROM_LAST) { + qsIterator.remove(); + log.info("Not consume long time, Clean order info, {}:{}, {}", topicAtGroup, entry.getValue(), topicConfig); + continue; + } + } + } + } + + @Override + public String encode() { + return this.encode(false); + } + + @Override + public String configFilePath() { + if (brokerController != null) { + return BrokerPathConfigHelper.getConsumerOrderInfoPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir()); + } else { + return BrokerPathConfigHelper.getConsumerOrderInfoPath("~"); + } + } + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + ConsumerOrderInfoManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOrderInfoManager.class); + if (obj != null) { + this.table = obj.table; + } + } + } + + @Override + public String encode(boolean prettyFormat) { + this.autoClean(); + + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("{\n").append("\t\"table\":{"); + Iterator>> iterator = + this.table.entrySet().iterator(); + int count1 = 0; + while (iterator.hasNext()) { + Map.Entry> entry = + iterator.next(); + if (count1 > 0) { + stringBuilder.append(","); + } + stringBuilder.append("\n\t\t\"").append(entry.getKey()).append("\":{"); + Iterator> qsIterator = entry.getValue().entrySet().iterator(); + int count2 = 0; + while (qsIterator.hasNext()) { + Map.Entry qsEntry = qsIterator.next(); + if (count2 > 0) { + stringBuilder.append(","); + } + stringBuilder.append("\n\t\t\t").append(qsEntry.getKey()).append(":") + .append(qsEntry.getValue().encode()); + count2++; + } + stringBuilder.append("\n\t\t}"); + count1++; + } + stringBuilder.append("\n\t}").append("\n}"); + return stringBuilder.toString(); + } + + public static class OrderInfo { + /** + * offset + */ + private List offsetList; + /** + * consumed count + */ + private int consumedCount; + /** + * last consume timestamp + */ + private long lastConsumeTimestamp; + /** + * commit offset bit + */ + private long commitOffsetBit; + + public OrderInfo() { + } + + public List getOffsetList() { + return offsetList; + } + + public void setOffsetList(List offsetList) { + this.offsetList = offsetList; + } + + public static List simpleO(List offsetList) { + List simple = new ArrayList<>(); + if (offsetList.size() == 1) { + simple.addAll(offsetList); + return simple; + } + Long first = offsetList.get(0); + simple.add(first); + for (int i = 1; i < offsetList.size(); i++) { + simple.add(offsetList.get(i) - first); + } + return simple; + } + + public int getConsumedCount() { + return consumedCount; + } + + public void setConsumedCount(int consumedCount) { + this.consumedCount = consumedCount; + } + + public long getLastConsumeTimestamp() { + return lastConsumeTimestamp; + } + + public void setLastConsumeTimestamp(long lastConsumeTimestamp) { + this.lastConsumeTimestamp = lastConsumeTimestamp; + } + + public long getCommitOffsetBit() { + return commitOffsetBit; + } + + public void setCommitOffsetBit(long commitOffsetBit) { + this.commitOffsetBit = commitOffsetBit; + } + + @JSONField(serialize = false, deserialize = false) + public boolean isDone() { + if (offsetList == null || offsetList.isEmpty()) { + return true; + } + int num = offsetList.size(); + for (byte i = 0; i < num; i++) { + if ((commitOffsetBit & (1L << i)) == 0) { + return false; + } + } + return true; + } + + @JSONField(serialize = false, deserialize = false) + public String encode() { + StringBuilder sb = new StringBuilder(); + sb.append("{").append("\"c\":").append(getConsumedCount()); + sb.append(",").append("\"cm\":").append(getCommitOffsetBit()); + sb.append(",").append("\"l\":").append(getLastConsumeTimestamp()); + sb.append(",").append("\"o\":["); + if (getOffsetList() != null) { + for (int i = 0; i < getOffsetList().size(); i++) { + sb.append(getOffsetList().get(i)); + if (i < getOffsetList().size() - 1) { + sb.append(","); + } + } + } + sb.append("]").append("}"); + return sb.toString(); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OrderInfo"); + sb.append("@").append(this.hashCode()); + sb.append("{offsetList=").append(offsetList); + sb.append(", consumedCount=").append(consumedCount); + sb.append(", lastConsumeTimestamp=").append(lastConsumeTimestamp); + sb.append(", commitOffsetBit=").append(commitOffsetBit); + sb.append(", isDone=").append(isDone()); + sb.append('}'); + return sb.toString(); + } + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index de7f3fce81c7a70a7e9b0eeb1daa6a2f9dd68c4c..252201a058ed4514cf0361d169deb640ba141958 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -30,8 +30,6 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.namesrv.TopAddressing; import org.apache.rocketmq.common.protocol.RequestCode; @@ -41,15 +39,20 @@ import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionResponseHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader; import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.RemotingClient; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; @@ -394,4 +397,39 @@ public class BrokerOuterAPI { public void registerRPCHook(RPCHook rpcHook) { remotingClient.registerRPCHook(rpcHook); } + + public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException { + return getTopicRouteInfoFromNameServer(topic, timeoutMillis, true); + } + + public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis, + boolean allowTopicNotExist) throws MQBrokerException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader(); + requestHeader.setTopic(topic); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.TOPIC_NOT_EXIST: { + if (allowTopicNotExist) { + log.warn("get Topic [{}] RouteInfoFromNameServer is not exist value", topic); + } + + break; + } + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + return TopicRouteData.decode(body, TopicRouteData.class); + } + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java new file mode 100644 index 0000000000000000000000000000000000000000..29f750784c0822df56e5d1f7ab7e39fad426e99c --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import com.alibaba.fastjson.JSON; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.util.MsgUtil; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.PopAckConstants; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.pop.AckMsg; + +public class AckMessageProcessor implements NettyRequestProcessor { + private static final InternalLogger POP_LOGGER = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME); + private final BrokerController brokerController; + private String reviveTopic; + private PopReviveService[] popReviveServices; + + public AckMessageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + this.reviveTopic = PopAckConstants.REVIVE_TOPIC + this.brokerController.getBrokerConfig().getBrokerClusterName(); + this.popReviveServices = new PopReviveService[this.brokerController.getBrokerConfig().getReviveQueueNum()]; + for (int i = 0; i < this.brokerController.getBrokerConfig().getReviveQueueNum(); i++) { + this.popReviveServices[i] = new PopReviveService(i, brokerController, reviveTopic); + } + } + + @Override + public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + return this.processRequest(ctx.channel(), request, true); + } + + @Override + public boolean rejectRequest() { + return false; + } + + public void startPopReviveService() { + for (PopReviveService popReviveService : popReviveServices) { + popReviveService.start(); + } + } + + public void shutdownPopReviveService() { + for (PopReviveService popReviveService : popReviveServices) { + popReviveService.shutdown(); + } + } + + private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend) throws RemotingCommandException { + final AckMessageRequestHeader requestHeader = (AckMessageRequestHeader) request.decodeCommandCustomHeader(AckMessageRequestHeader.class); + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + AckMsg ackMsg = new AckMsg(); + RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null); + response.setOpaque(request.getOpaque()); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + if (null == topicConfig) { + POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel)); + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL))); + return response; + } + + if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) { + String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]", + requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); + POP_LOGGER.warn(errorInfo); + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark(errorInfo); + return response; + } + long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) { + response.setCode(ResponseCode.NO_MESSAGE); + return response; + } + String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo()); + + ackMsg.setAckOffset(requestHeader.getOffset()); + ackMsg.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfo)); + ackMsg.setConsumerGroup(requestHeader.getConsumerGroup()); + ackMsg.setTopic(requestHeader.getTopic()); + ackMsg.setQueueId(requestHeader.getQueueId()); + ackMsg.setPopTime(ExtraInfoUtil.getPopTime(extraInfo)); + + int rqId = ExtraInfoUtil.getReviveQid(extraInfo); + + if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) { + // order + String lockKey = requestHeader.getTopic() + PopAckConstants.SPLIT + + requestHeader.getConsumerGroup() + PopAckConstants.SPLIT + requestHeader.getQueueId(); + long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(), + requestHeader.getTopic(), requestHeader.getQueueId()); + if (requestHeader.getOffset() < oldOffset) { + return response; + } + while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) { + } + try { + oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(), + requestHeader.getTopic(), requestHeader.getQueueId()); + if (requestHeader.getOffset() < oldOffset) { + return response; + } + long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext( + requestHeader.getTopic(), requestHeader.getConsumerGroup(), + requestHeader.getQueueId(), requestHeader.getOffset()); + if (nextOffset > -1) { + this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), + requestHeader.getConsumerGroup(), requestHeader.getTopic(), + requestHeader.getQueueId(), + nextOffset); + this.brokerController.getPopMessageProcessor().notifyMessageArriving(requestHeader.getTopic(), requestHeader.getConsumerGroup(), + requestHeader.getQueueId()); + } else if (nextOffset == -1) { + String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s", + lockKey, oldOffset, requestHeader.getOffset(), nextOffset, channel.remoteAddress()); + POP_LOGGER.warn(errorInfo); + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark(errorInfo); + return response; + } + } finally { + this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey); + } + return response; + } + + if (this.brokerController.getPopMessageProcessor().getPopBufferMergeService().addAk(rqId, ackMsg)) { + return response; + } + + msgInner.setTopic(reviveTopic); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + //msgInner.setQueueId(Integer.valueOf(extraInfo[3])); + msgInner.setQueueId(rqId); + msgInner.setTags(PopAckConstants.ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); + msgInner.setBornHost(this.brokerController.getStoreHost()); + msgInner.setStoreHost(this.brokerController.getStoreHost()); + MsgUtil.setMessageDeliverTime(this.brokerController, msgInner, ExtraInfoUtil.getPopTime(extraInfo) + ExtraInfoUtil.getInvisibleTime(extraInfo)); + msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg)); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { + POP_LOGGER.error("put ack msg error:" + putMessageResult); + } + return response; + } + +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java new file mode 100644 index 0000000000000000000000000000000000000000..e40a5e8d901e030b3b261d66dda505c05475f583 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import com.alibaba.fastjson.JSON; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.util.MsgUtil; +import org.apache.rocketmq.common.PopAckConstants; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeResponseHeader; +import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.pop.AckMsg; +import org.apache.rocketmq.store.pop.PopCheckPoint; + +public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { + private static final InternalLogger POP_LOGGER = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME); + private final BrokerController brokerController; + private String reviveTopic; + + public ChangeInvisibleTimeProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + this.reviveTopic = PopAckConstants.REVIVE_TOPIC + this.brokerController.getBrokerConfig().getBrokerClusterName(); + + } + + @Override + public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + return this.processRequest(ctx.channel(), request, true); + } + + @Override + public boolean rejectRequest() { + return false; + } + + private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend) throws RemotingCommandException { + final ChangeInvisibleTimeRequestHeader requestHeader = (ChangeInvisibleTimeRequestHeader) request.decodeCommandCustomHeader(ChangeInvisibleTimeRequestHeader.class); + RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + final ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.readCustomHeader(); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + if (null == topicConfig) { + POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel)); + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL))); + return response; + } + + if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) { + String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]", + requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); + POP_LOGGER.warn(errorInfo); + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark(errorInfo); + return response; + } + long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) { + response.setCode(ResponseCode.NO_MESSAGE); + return response; + } + + String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo()); + + // add new ck + long now = System.currentTimeMillis(); + PutMessageResult ckResult = appendCheckPoint(requestHeader, ExtraInfoUtil.getReviveQid(extraInfo), requestHeader.getQueueId(), requestHeader.getOffset(), now); + + if (ckResult.getPutMessageStatus() != PutMessageStatus.PUT_OK + && ckResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT + && ckResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT + && ckResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { + POP_LOGGER.error("change Invisible, put new ck error: {}", ckResult); + response.setCode(ResponseCode.SYSTEM_ERROR); + return response; + } + + // ack old msg. + try { + ackOrigin(requestHeader, extraInfo); + } catch (Throwable e) { + POP_LOGGER.error("change Invisible, put ack msg error: {}, {}", requestHeader.getExtraInfo(), e.getMessage()); + // cancel new ck? + } + + responseHeader.setInvisibleTime(requestHeader.getInvisibleTime()); + responseHeader.setPopTime(now); + responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo)); + return response; + } + + private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, String[] extraInfo) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + AckMsg ackMsg = new AckMsg(); + + ackMsg.setAckOffset(requestHeader.getOffset()); + ackMsg.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfo)); + ackMsg.setConsumerGroup(requestHeader.getConsumerGroup()); + ackMsg.setTopic(requestHeader.getTopic()); + ackMsg.setQueueId(requestHeader.getQueueId()); + ackMsg.setPopTime(ExtraInfoUtil.getPopTime(extraInfo)); + + int rqId = ExtraInfoUtil.getReviveQid(extraInfo); + + if (brokerController.getPopMessageProcessor().getPopBufferMergeService().addAk(rqId, ackMsg)) { + return; + } + + msgInner.setTopic(reviveTopic); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + msgInner.setQueueId(rqId); + msgInner.setTags(PopAckConstants.ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); + msgInner.setBornHost(this.brokerController.getStoreHost()); + msgInner.setStoreHost(this.brokerController.getStoreHost()); + MsgUtil.setMessageDeliverTime(this.brokerController, msgInner, ExtraInfoUtil.getPopTime(extraInfo) + ExtraInfoUtil.getInvisibleTime(extraInfo)); + msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg)); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { + POP_LOGGER.error("change Invisible, put ack msg fail: {}, {}", ackMsg, putMessageResult); + } + } + + private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader requestHeader, int reviveQid, int queueId, long offset, long popTime) { + // add check point msg to revive log + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(reviveTopic); + PopCheckPoint ck = new PopCheckPoint(); + ck.setBitMap(0); + ck.setNum((byte) 1); + ck.setPopTime(popTime); + ck.setInvisibleTime(requestHeader.getInvisibleTime()); + ck.getStartOffset(offset); + ck.setCId(requestHeader.getConsumerGroup()); + ck.setTopic(requestHeader.getTopic()); + ck.setQueueId((byte) queueId); + ck.addDiff(0); + + msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); + msgInner.setQueueId(reviveQid); + msgInner.setTags(PopAckConstants.CK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); + msgInner.setBornHost(this.brokerController.getStoreHost()); + msgInner.setStoreHost(this.brokerController.getStoreHost()); + MsgUtil.setMessageDeliverTime(this.brokerController, msgInner, ck.getReviveTime() - PopAckConstants.ackTimeInterval); + msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genCkUniqueId(ck)); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("change Invisible , appendCheckPoint, topic {}, queueId {},reviveId {}, cid {}, startOffset {}, rt {}, result {}", requestHeader.getTopic(), queueId, reviveQid, requestHeader.getConsumerGroup(), offset, + ck.getReviveTime(), putMessageResult); + } + + return putMessageResult; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java new file mode 100644 index 0000000000000000000000000000000000000000..615a70e4c91899d47689d33678d18578818d9669 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java @@ -0,0 +1,731 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import com.alibaba.fastjson.JSON; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.util.MsgUtil; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.PopAckConstants; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.pop.AckMsg; +import org.apache.rocketmq.store.pop.PopCheckPoint; + +public class PopBufferMergeService extends ServiceThread { + private static final InternalLogger POP_LOGGER = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME); + ConcurrentHashMap + buffer = new ConcurrentHashMap<>(1024 * 16); + ConcurrentHashMap> commitOffsets = + new ConcurrentHashMap<>(); + private volatile boolean serving = true; + private AtomicInteger counter = new AtomicInteger(0); + private int scanTimes = 0; + private final BrokerController brokerController; + private final PopMessageProcessor popMessageProcessor; + private final PopMessageProcessor.QueueLockManager queueLockManager; + private final long interval = 5; + private final long minute5 = 5 * 60 * 1000; + private final int countOfMinute1 = (int) (60 * 1000 / interval); + private final int countOfSecond1 = (int) (1000 / interval); + private final int countOfSecond30 = (int) (30 * 1000 / interval); + + private volatile boolean master = false; + + public PopBufferMergeService(BrokerController brokerController, PopMessageProcessor popMessageProcessor) { + super(); + this.brokerController = brokerController; + this.popMessageProcessor = popMessageProcessor; + this.queueLockManager = popMessageProcessor.getQueueLockManager(); + } + + private boolean checkAndSetMaster() { + this.master = brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE; + return this.master; + } + + @Override + public String getServiceName() { + return "PopBufferMergeService"; + } + + @Override + public void run() { + // scan + while (!this.isStopped()) { + try { + if (!checkAndSetMaster()) { + // slave + this.waitForRunning(interval * 200 * 5); + POP_LOGGER.info("Broker is {}, {}, clear all data", + brokerController.getMessageStoreConfig().getBrokerRole(), this.master); + this.buffer.clear(); + this.commitOffsets.clear(); + continue; + } + + scan(); + if (scanTimes % countOfSecond30 == 0) { + scanGarbage(); + } + + this.waitForRunning(interval); + + if (!this.serving && this.buffer.size() == 0 && totalSize() == 0) { + this.serving = true; + } + } catch (Throwable e) { + POP_LOGGER.error("PopBufferMergeService error", e); + this.waitForRunning(3000); + } + } + + this.serving = false; + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + } + if (!checkAndSetMaster()) { + return; + } + while (this.buffer.size() > 0 || totalSize() > 0) { + scan(); + } + } + + private int scanCommitOffset() { + Iterator>> iterator = this.commitOffsets.entrySet().iterator(); + int count = 0; + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + LinkedBlockingDeque queue = entry.getValue().get(); + PopCheckPointWrapper pointWrapper; + while ((pointWrapper = queue.peek()) != null) { + // 1. just offset & stored, not processed by scan + // 2. ck is buffer(acked) + // 3. ck is buffer(not all acked), all ak are stored and ck is stored + if ((pointWrapper.isJustOffset() && pointWrapper.isCkStored()) || isCkDone(pointWrapper) + || (isCkDoneForFinish(pointWrapper) && pointWrapper.isCkStored())) { + if (commitOffset(pointWrapper)) { + queue.poll(); + } else { + break; + } + } else { + if (System.currentTimeMillis() - pointWrapper.getCk().getPopTime() + > brokerController.getBrokerConfig().getPopCkStayBufferTime() * 2) { + POP_LOGGER.warn("[PopBuffer] ck offset long time not commit, {}", pointWrapper); + } + break; + } + } + final int qs = queue.size(); + count += qs; + if (qs > 5000 && scanTimes % countOfSecond1 == 0) { + POP_LOGGER.info("[PopBuffer] offset queue size too long, {}, {}", + entry.getKey(), qs); + } + } + return count; + } + + public long getLatestOffset(String lockKey) { + QueueWithTime queue = this.commitOffsets.get(lockKey); + if (queue == null) { + return -1; + } + PopCheckPointWrapper pointWrapper = queue.get().peekLast(); + if (pointWrapper != null) { + return pointWrapper.getNextBeginOffset(); + } + return -1; + } + + public long getLatestOffset(String topic, String group, int queueId) { + return getLatestOffset(KeyBuilder.buildPollingKey(topic, group, queueId)); + } + + private void scanGarbage() { + Iterator>> iterator = commitOffsets.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + if (entry.getKey() == null) { + continue; + } + String[] keyArray = entry.getKey().split(PopAckConstants.SPLIT); + if (keyArray == null || keyArray.length != 3) { + continue; + } + String topic = keyArray[0]; + String cid = keyArray[1]; + if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) { + POP_LOGGER.info("[PopBuffer]remove not exit topic {} in buffer!", topic); + iterator.remove(); + continue; + } + if (!brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().containsKey(cid)) { + POP_LOGGER.info("[PopBuffer]remove not exit sub {} of topic {} in buffer!", cid, topic); + iterator.remove(); + continue; + } + if (System.currentTimeMillis() - entry.getValue().getTime() > minute5) { + POP_LOGGER.info("[PopBuffer]remove long time not used sub {} of topic {} in buffer!", cid, topic); + iterator.remove(); + continue; + } + } + } + + private void scan() { + long startTime = System.currentTimeMillis(); + int count = 0, countCk = 0; + Iterator> iterator = buffer.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + PopCheckPointWrapper pointWrapper = entry.getValue(); + + // just process offset(already stored at pull thread), or buffer ck(not stored and ack finish) + if ((pointWrapper.isJustOffset() && pointWrapper.isCkStored()) || isCkDone(pointWrapper) + || (isCkDoneForFinish(pointWrapper) && pointWrapper.isCkStored())) { + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("[PopBuffer]ck done, {}", pointWrapper); + } + iterator.remove(); + counter.decrementAndGet(); + continue; + } + + PopCheckPoint point = pointWrapper.getCk(); + long now = System.currentTimeMillis(); + + boolean removeCk = !this.serving; + // ck will be timeout + if (point.getReviveTime() - now < brokerController.getBrokerConfig().getPopCkStayBufferTimeOut()) { + removeCk = true; + } + + // the time stayed is too long + if (now - point.getPopTime() > brokerController.getBrokerConfig().getPopCkStayBufferTime()) { + removeCk = true; + } + + if (now - point.getPopTime() > brokerController.getBrokerConfig().getPopCkStayBufferTime() * 2L) { + POP_LOGGER.warn("[PopBuffer]ck finish fail, stay too long, {}", pointWrapper); + } + + // double check + if (isCkDone(pointWrapper)) { + continue; + } else if (pointWrapper.isJustOffset()) { + // just offset should be in store. + if (pointWrapper.getReviveQueueOffset() < 0) { + putCkToStore(pointWrapper, false); + countCk++; + } + continue; + } else if (removeCk) { + // put buffer ak to store + if (pointWrapper.getReviveQueueOffset() < 0) { + putCkToStore(pointWrapper, false); + countCk++; + } + + if (!pointWrapper.isCkStored()) { + continue; + } + + for (byte i = 0; i < point.getNum(); i++) { + // reput buffer ak to store + if (DataConverter.getBit(pointWrapper.getBits().get(), i) + && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) { + if (putAckToStore(pointWrapper, i)) { + count++; + markBitCAS(pointWrapper.getToStoreBits(), i); + } + } + } + + if (isCkDoneForFinish(pointWrapper) && pointWrapper.isCkStored()) { + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("[PopBuffer]ck finish, {}", pointWrapper); + } + iterator.remove(); + counter.decrementAndGet(); + continue; + } + } + } + + int offsetBufferSize = scanCommitOffset(); + + long eclipse = System.currentTimeMillis() - startTime; + if (eclipse > brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() - 1000) { + POP_LOGGER.warn("[PopBuffer]scan stop, because eclipse too long, PopBufferEclipse={}, " + + "PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}", + eclipse, count, countCk, counter.get(), offsetBufferSize); + this.serving = false; + } else { + if (scanTimes % countOfSecond1 == 0) { + POP_LOGGER.info("[PopBuffer]scan, PopBufferEclipse={}, " + + "PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}", + eclipse, count, countCk, counter.get(), offsetBufferSize); + } + } + scanTimes++; + + if (scanTimes >= countOfMinute1) { + counter.set(this.buffer.size()); + scanTimes = 0; + } + } + + private int totalSize() { + int count = 0; + Iterator>> iterator = this.commitOffsets.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + LinkedBlockingDeque queue = entry.getValue().get(); + count += queue.size(); + } + return count; + } + + private void markBitCAS(AtomicInteger setBits, int index) { + while (true) { + int bits = setBits.get(); + if (DataConverter.getBit(bits, index)) { + break; + } + + int newBits = DataConverter.setBit(bits, index, true); + if (setBits.compareAndSet(bits, newBits)) { + break; + } + } + } + + private boolean commitOffset(final PopCheckPointWrapper wrapper) { + if (wrapper.getNextBeginOffset() < 0) { + return true; + } + + final PopCheckPoint popCheckPoint = wrapper.getCk(); + final String lockKey = wrapper.getLockKey(); + + if (!queueLockManager.tryLock(lockKey)) { + return false; + } + try { + final long offset = brokerController.getConsumerOffsetManager().queryOffset(popCheckPoint.getCId(), popCheckPoint.getTopic(), popCheckPoint.getQueueId()); + if (wrapper.getNextBeginOffset() > offset) { + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("Commit offset, {}, {}", wrapper, offset); + } + } else { + // maybe store offset is not correct. + POP_LOGGER.warn("Commit offset, consumer offset less than store, {}, {}", wrapper, offset); + } + brokerController.getConsumerOffsetManager().commitOffset(getServiceName(), + popCheckPoint.getCId(), popCheckPoint.getTopic(), popCheckPoint.getQueueId(), wrapper.getNextBeginOffset()); + } finally { + queueLockManager.unLock(lockKey); + } + return true; + } + + private boolean putOffsetQueue(PopCheckPointWrapper pointWrapper) { + QueueWithTime queue = this.commitOffsets.get(pointWrapper.getLockKey()); + if (queue == null) { + queue = new QueueWithTime<>(); + QueueWithTime old = this.commitOffsets.putIfAbsent(pointWrapper.getLockKey(), queue); + if (old != null) { + queue = old; + } + } + queue.setTime(pointWrapper.getCk().getPopTime()); + return queue.get().offer(pointWrapper); + } + + private boolean checkQueueOk(PopCheckPointWrapper pointWrapper) { + QueueWithTime queue = this.commitOffsets.get(pointWrapper.getLockKey()); + if (queue == null) { + return true; + } + return queue.get().size() < brokerController.getBrokerConfig().getPopCkOffsetMaxQueueSize(); + } + + /** + * put to store && add to buffer. + * @param point + * @param reviveQueueId + * @param reviveQueueOffset + * @param nextBeginOffset + * @return + */ + public void addCkJustOffset(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset, long nextBeginOffset) { + PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, reviveQueueOffset, point, nextBeginOffset, true); + + this.putCkToStore(pointWrapper, !checkQueueOk(pointWrapper)); + + putOffsetQueue(pointWrapper); + this.buffer.put(pointWrapper.getMergeKey(), pointWrapper); + this.counter.incrementAndGet(); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("[PopBuffer]add ck just offset, {}", pointWrapper); + } + } + + public void addCkMock(String group, String topic, int queueId, long startOffset, long invisibleTime, + long popTime, int reviveQueueId, long nextBeginOffset) { + final PopCheckPoint ck = new PopCheckPoint(); + ck.setBitMap(0); + ck.setNum((byte) 0); + ck.setPopTime(popTime); + ck.setInvisibleTime(invisibleTime); + ck.getStartOffset(startOffset); + ck.setCId(group); + ck.setTopic(topic); + ck.setQueueId((byte) queueId); + + PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, Long.MAX_VALUE, ck, nextBeginOffset, true); + pointWrapper.setCkStored(true); + + putOffsetQueue(pointWrapper); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("[PopBuffer]add ck just offset, mocked, {}", pointWrapper); + } + } + + public boolean addCk(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset, long nextBeginOffset) { + // key: point.getT() + point.getC() + point.getQ() + point.getSo() + point.getPt() + if (!brokerController.getBrokerConfig().isEnablePopBufferMerge()) { + return false; + } + if (!serving) { + return false; + } + + long now = System.currentTimeMillis(); + if (point.getReviveTime() - now < brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() + 1500) { + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.warn("[PopBuffer]add ck, timeout, {}, {}", point, now); + } + return false; + } + + if (this.counter.get() > brokerController.getBrokerConfig().getPopCkMaxBufferSize()) { + POP_LOGGER.warn("[PopBuffer]add ck, max size, {}, {}", point, this.counter.get()); + return false; + } + + PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, reviveQueueOffset, point, nextBeginOffset); + + if (!checkQueueOk(pointWrapper)) { + return false; + } + + putOffsetQueue(pointWrapper); + this.buffer.put(pointWrapper.getMergeKey(), pointWrapper); + this.counter.incrementAndGet(); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("[PopBuffer]add ck, {}", pointWrapper); + } + return true; + } + + public boolean addAk(int reviveQid, AckMsg ackMsg) { + if (!brokerController.getBrokerConfig().isEnablePopBufferMerge()) { + return false; + } + if (!serving) { + return false; + } + try { + PopCheckPointWrapper pointWrapper = this.buffer.get(ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime()); + if (pointWrapper == null) { + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.warn("[PopBuffer]add ack fail, rqId={}, no ck, {}", reviveQid, ackMsg); + } + return false; + } + + if (pointWrapper.isJustOffset()) { + return false; + } + + PopCheckPoint point = pointWrapper.getCk(); + long now = System.currentTimeMillis(); + + if (point.getReviveTime() - now < brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() + 1500) { + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.warn("[PopBuffer]add ack fail, rqId={}, almost timeout for revive, {}, {}, {}", reviveQid, pointWrapper, ackMsg, now); + } + return false; + } + + if (now - point.getPopTime() > brokerController.getBrokerConfig().getPopCkStayBufferTime() - 1500) { + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.warn("[PopBuffer]add ack fail, rqId={}, stay too long, {}, {}, {}", reviveQid, pointWrapper, ackMsg, now); + } + return false; + } + + int indexOfAck = point.indexOfAck(ackMsg.getAckOffset()); + if (indexOfAck > -1) { + markBitCAS(pointWrapper.getBits(), indexOfAck); + } else { + POP_LOGGER.error("[PopBuffer]Invalid index of ack, reviveQid={}, {}, {}", reviveQid, ackMsg, point); + return true; + } + + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("[PopBuffer]add ack, rqId={}, {}, {}", reviveQid, pointWrapper, ackMsg); + } + +// // check ak done +// if (isCkDone(pointWrapper)) { +// // cancel ck for timer +// cancelCkTimer(pointWrapper); +// } + return true; + } catch (Throwable e) { + POP_LOGGER.error("[PopBuffer]add ack error, rqId=" + reviveQid + ", " + ackMsg, e); + } + + return false; + } + + private void putCkToStore(final PopCheckPointWrapper pointWrapper, final boolean runInCurrent) { + if (pointWrapper.getReviveQueueOffset() >= 0) { + return; + } + MessageExtBrokerInner msgInner = popMessageProcessor.buildCkMsg(pointWrapper.getCk(), pointWrapper.getReviveQueueId()); + PutMessageResult putMessageResult = brokerController.getMessageStore().putMessage(msgInner); + if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { + POP_LOGGER.error("[PopBuffer]put ck to store fail: {}, {}", pointWrapper, putMessageResult); + return; + } + pointWrapper.setCkStored(true); + pointWrapper.setReviveQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset()); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("[PopBuffer]put ck to store ok: {}, {}", pointWrapper, putMessageResult); + } + } + + private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgIndex) { + PopCheckPoint point = pointWrapper.getCk(); + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + final AckMsg ackMsg = new AckMsg(); + + ackMsg.setAckOffset(point.ackOffsetByIndex(msgIndex)); + ackMsg.setStartOffset(point.getStartOffset()); + ackMsg.setConsumerGroup(point.getCId()); + ackMsg.setTopic(point.getTopic()); + ackMsg.setQueueId(point.getQueueId()); + ackMsg.setPopTime(point.getPopTime()); + msgInner.setTopic(popMessageProcessor.reviveTopic); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); + msgInner.setQueueId(pointWrapper.getReviveQueueId()); + msgInner.setTags(PopAckConstants.ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); + msgInner.setBornHost(brokerController.getStoreHost()); + msgInner.setStoreHost(brokerController.getStoreHost()); + + MsgUtil.setMessageDeliverTime(this.brokerController, msgInner, point.getReviveTime()); + msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg)); + + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + PutMessageResult putMessageResult = brokerController.getMessageStore().putMessage(msgInner); + if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { + POP_LOGGER.error("[PopBuffer]put ack to store fail: {}, {}, {}", pointWrapper, ackMsg, putMessageResult); + return false; + } + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("[PopBuffer]put ack to store ok: {}, {}, {}", pointWrapper, ackMsg, putMessageResult); + } + + return true; + } + + private boolean isCkDone(PopCheckPointWrapper pointWrapper) { + byte num = pointWrapper.getCk().getNum(); + for (byte i = 0; i < num; i++) { + if (!DataConverter.getBit(pointWrapper.getBits().get(), i)) { + return false; + } + } + return true; + } + + private boolean isCkDoneForFinish(PopCheckPointWrapper pointWrapper) { + byte num = pointWrapper.getCk().getNum(); + int bits = pointWrapper.getBits().get() ^ pointWrapper.getToStoreBits().get(); + for (byte i = 0; i < num; i++) { + if (DataConverter.getBit(bits, i)) { + return false; + } + } + return true; + } + + public class QueueWithTime { + private final LinkedBlockingDeque queue; + private long time; + + public QueueWithTime() { + this.queue = new LinkedBlockingDeque<>(); + this.time = System.currentTimeMillis(); + } + + public void setTime(long popTime) { + this.time = popTime; + } + + public long getTime() { + return time; + } + + public LinkedBlockingDeque get() { + return queue; + } + } + + public class PopCheckPointWrapper { + private final int reviveQueueId; + // -1: not stored, >=0: stored, Long.MAX: storing. + private volatile long reviveQueueOffset; + private final PopCheckPoint ck; + // bit for concurrent + private final AtomicInteger bits; + // bit for stored buffer ak + private final AtomicInteger toStoreBits; + private final long nextBeginOffset; + private final String lockKey; + private final String mergeKey; + private final boolean justOffset; + private volatile boolean ckStored = false; + + public PopCheckPointWrapper(int reviveQueueId, long reviveQueueOffset, PopCheckPoint point, long nextBeginOffset) { + this.reviveQueueId = reviveQueueId; + this.reviveQueueOffset = reviveQueueOffset; + this.ck = point; + this.bits = new AtomicInteger(0); + this.toStoreBits = new AtomicInteger(0); + this.nextBeginOffset = nextBeginOffset; + this.lockKey = ck.getTopic() + PopAckConstants.SPLIT + ck.getCId() + PopAckConstants.SPLIT + ck.getQueueId(); + this.mergeKey = point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime(); + this.justOffset = false; + } + + public PopCheckPointWrapper(int reviveQueueId, long reviveQueueOffset, PopCheckPoint point, long nextBeginOffset, + boolean justOffset) { + this.reviveQueueId = reviveQueueId; + this.reviveQueueOffset = reviveQueueOffset; + this.ck = point; + this.bits = new AtomicInteger(0); + this.toStoreBits = new AtomicInteger(0); + this.nextBeginOffset = nextBeginOffset; + this.lockKey = ck.getTopic() + PopAckConstants.SPLIT + ck.getCId() + PopAckConstants.SPLIT + ck.getQueueId(); + this.mergeKey = point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime(); + this.justOffset = justOffset; + } + + public int getReviveQueueId() { + return reviveQueueId; + } + + public long getReviveQueueOffset() { + return reviveQueueOffset; + } + + public boolean isCkStored() { + return ckStored; + } + + public void setReviveQueueOffset(long reviveQueueOffset) { + this.reviveQueueOffset = reviveQueueOffset; + } + + public PopCheckPoint getCk() { + return ck; + } + + public AtomicInteger getBits() { + return bits; + } + + public AtomicInteger getToStoreBits() { + return toStoreBits; + } + + public long getNextBeginOffset() { + return nextBeginOffset; + } + + public String getLockKey() { + return lockKey; + } + + public String getMergeKey() { + return mergeKey; + } + + public boolean isJustOffset() { + return justOffset; + } + + public void setCkStored(boolean ckStored) { + this.ckStored = ckStored; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CkWrap{"); + sb.append("rq=").append(reviveQueueId); + sb.append(", rqo=").append(reviveQueueOffset); + sb.append(", ck=").append(ck); + sb.append(", bits=").append(bits); + sb.append(", sBits=").append(toStoreBits); + sb.append(", nbo=").append(nextBeginOffset); + sb.append(", cks=").append(ckStored); + sb.append(", jo=").append(justOffset); + sb.append('}'); + return sb.toString(); + } + } + +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java new file mode 100644 index 0000000000000000000000000000000000000000..aa97fc84d057304b2ac60cc4fe5d1cb75b4203d5 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -0,0 +1,967 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import com.alibaba.fastjson.JSON; +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.FileRegion; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.filter.ConsumerFilterData; +import org.apache.rocketmq.broker.filter.ConsumerFilterManager; +import org.apache.rocketmq.broker.filter.ExpressionMessageFilter; +import org.apache.rocketmq.broker.longpolling.PopRequest; +import org.apache.rocketmq.broker.pagecache.ManyMessageTransfer; +import org.apache.rocketmq.broker.util.MsgUtil; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.PopAckConstants; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.ConsumeInitMode; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.common.filter.FilterAPI; +import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.PopMessageResponseHeader; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.netty.RequestTask; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.GetMessageStatus; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.pop.AckMsg; +import org.apache.rocketmq.store.pop.PopCheckPoint; + +public class PopMessageProcessor implements NettyRequestProcessor { + private static final InternalLogger POP_LOGGER = + InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME); + private final BrokerController brokerController; + private Random random = new Random(System.currentTimeMillis()); + String reviveTopic; + private static final String BORN_TIME = "bornTime"; + + private static final int POLLING_SUC = 0; + private static final int POLLING_FULL = 1; + private static final int POLLING_TIMEOUT = 2; + private static final int NOT_POLLING = 3; + + private ConcurrentHashMap> topicCidMap; + private ConcurrentLinkedHashMap> pollingMap; + private AtomicLong totalPollingNum = new AtomicLong(0); + private PopLongPollingService popLongPollingService; + private PopBufferMergeService popBufferMergeService; + private QueueLockManager queueLockManager; + + public PopMessageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + this.reviveTopic = + PopAckConstants.REVIVE_TOPIC + this.brokerController.getBrokerConfig().getBrokerClusterName(); + // 100000 topic default, 100000 lru topic + cid + qid + this.topicCidMap = new ConcurrentHashMap<>(this.brokerController.getBrokerConfig().getPopPollingMapSize()); + this.pollingMap = new ConcurrentLinkedHashMap.Builder>() + .maximumWeightedCapacity(this.brokerController.getBrokerConfig().getPopPollingMapSize()).build(); + this.popLongPollingService = new PopLongPollingService(); + this.queueLockManager = new QueueLockManager(); + this.popBufferMergeService = new PopBufferMergeService(this.brokerController, this); + } + + public PopLongPollingService getPopLongPollingService() { + return popLongPollingService; + } + + public PopBufferMergeService getPopBufferMergeService() { + return this.popBufferMergeService; + } + + public QueueLockManager getQueueLockManager() { + return queueLockManager; + } + + public static String genAckUniqueId(AckMsg ackMsg) { + return ackMsg.getTopic() + + PopAckConstants.SPLIT + ackMsg.getQueueId() + + PopAckConstants.SPLIT + ackMsg.getAckOffset() + + PopAckConstants.SPLIT + ackMsg.getConsumerGroup() + + PopAckConstants.SPLIT + ackMsg.getPopTime() + + PopAckConstants.SPLIT + PopAckConstants.ACK_TAG; + } + + public static String genCkUniqueId(PopCheckPoint ck) { + return ck.getTopic() + + PopAckConstants.SPLIT + ck.getQueueId() + + PopAckConstants.SPLIT + ck.getStartOffset() + + PopAckConstants.SPLIT + ck.getCId() + + PopAckConstants.SPLIT + ck.getPopTime() + + PopAckConstants.SPLIT + PopAckConstants.CK_TAG; + } + + @Override + public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + request.addExtField(BORN_TIME, String.valueOf(System.currentTimeMillis())); + return this.processRequest(ctx.channel(), request); + } + + @Override + public boolean rejectRequest() { + return false; + } + + public ConcurrentLinkedHashMap> getPollingMap() { + return pollingMap; + } + + public void notifyMessageArriving(final String topic, final int queueId) { + ConcurrentHashMap cids = topicCidMap.get(topic); + if (cids == null) { + return; + } + for (Entry cid : cids.entrySet()) { + if (queueId >= 0) { + notifyMessageArriving(topic, cid.getKey(), -1); + } + notifyMessageArriving(topic, cid.getKey(), queueId); + } + } + + public void notifyMessageArriving(final String topic, final String cid, final int queueId) { + ConcurrentSkipListSet remotingCommands = pollingMap.get(KeyBuilder.buildPollingKey(topic, cid, + queueId)); + if (remotingCommands == null || remotingCommands.isEmpty()) { + return; + } + PopRequest popRequest = remotingCommands.pollFirst(); + //clean inactive channel + while (popRequest != null && !popRequest.getChannel().isActive()) { + popRequest = remotingCommands.pollFirst(); + } + + if (popRequest == null) { + return; + } + totalPollingNum.decrementAndGet(); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("lock release , new msg arrive , wakeUp : {}", popRequest); + } + wakeUp(popRequest); + } + + private void wakeUp(final PopRequest request) { + if (request == null || !request.complete()) { + return; + } + Runnable run = new Runnable() { + @Override + public void run() { + try { + final RemotingCommand response = PopMessageProcessor.this.processRequest(request.getChannel(), + request.getRemotingCommand()); + + if (response != null) { + response.setOpaque(request.getRemotingCommand().getOpaque()); + response.markResponseType(); + try { + request.getChannel().writeAndFlush(response).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + if (!future.isSuccess()) { + POP_LOGGER.error("ProcessRequestWrapper response to {} failed", + future.channel().remoteAddress(), future.cause()); + POP_LOGGER.error(request.toString()); + POP_LOGGER.error(response.toString()); + } + } + }); + } catch (Throwable e) { + POP_LOGGER.error("ProcessRequestWrapper process request over, but response failed", e); + POP_LOGGER.error(request.toString()); + POP_LOGGER.error(response.toString()); + } + } + } catch (RemotingCommandException e1) { + POP_LOGGER.error("ExecuteRequestWhenWakeup run", e1); + } + } + }; + this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, request.getChannel(), + request.getRemotingCommand())); + } + + private RemotingCommand processRequest(final Channel channel, RemotingCommand request) + throws RemotingCommandException { + RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); + final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); + final PopMessageRequestHeader requestHeader = + (PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class); + StringBuilder startOffsetInfo = new StringBuilder(64); + StringBuilder msgOffsetInfo = new StringBuilder(64); + StringBuilder orderCountInfo = null; + if (requestHeader.isOrder()) { + orderCountInfo = new StringBuilder(64); + } + + response.setOpaque(request.getOpaque()); + + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("receive PopMessage request command, {}", request); + } + + if (requestHeader.isTimeoutTooMuch()) { + response.setCode(POLLING_TIMEOUT); + response.setRemark(String.format("the broker[%s] poping message is timeout too much", + this.brokerController.getBrokerConfig().getBrokerIP1())); + return response; + } + if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark(String.format("the broker[%s] poping message is forbidden", + this.brokerController.getBrokerConfig().getBrokerIP1())); + return response; + } + if (requestHeader.getMaxMsgNums() > 32) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("the broker[%s] poping message's num is greater than 32", + this.brokerController.getBrokerConfig().getBrokerIP1())); + return response; + } + + TopicConfig topicConfig = + this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + if (null == topicConfig) { + POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), + RemotingHelper.parseChannelRemoteAddr(channel)); + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL))); + return response; + } + + if (!PermName.isReadable(topicConfig.getPerm())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the topic[" + requestHeader.getTopic() + "] peeking message is forbidden"); + return response; + } + + if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) { + String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] " + + "consumer:[%s]", + requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), + channel.remoteAddress()); + POP_LOGGER.warn(errorInfo); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(errorInfo); + return response; + } + SubscriptionGroupConfig subscriptionGroupConfig = + this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup()); + if (null == subscriptionGroupConfig) { + response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); + response.setRemark(String.format("subscription group [%s] does not exist, %s", + requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST))); + return response; + } + ConsumerGroupInfo consumerGroupInfo = + this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup()); + if (null == consumerGroupInfo) { + POP_LOGGER.warn("the consumer's group info not exist, group: {}", requestHeader.getConsumerGroup()); + response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST); + response.setRemark("the consumer's group info not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC)); + return response; + } + + + if (!subscriptionGroupConfig.isConsumeEnable()) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("subscription group no permission, " + requestHeader.getConsumerGroup()); + return response; + } + + ExpressionMessageFilter messageFilter = null; + if (requestHeader.getExp() != null && requestHeader.getExp().length() > 0) { + try { + SubscriptionData subscriptionData = FilterAPI.build(requestHeader.getTopic(), requestHeader.getExp(), requestHeader.getExpType()); + ConsumerFilterData consumerFilterData = null; + if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) { + consumerFilterData = ConsumerFilterManager.build( + requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getExp(), + requestHeader.getExpType(), System.currentTimeMillis() + ); + if (consumerFilterData == null) { + POP_LOGGER.warn("Parse the consumer's subscription[{}] failed, group: {}", + requestHeader.getExp(), requestHeader.getConsumerGroup()); + response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED); + response.setRemark("parse the consumer's subscription failed"); + return response; + } + } + messageFilter = new ExpressionMessageFilter(subscriptionData, consumerFilterData, + brokerController.getConsumerFilterManager()); + } catch (Exception e) { + POP_LOGGER.warn("Parse the consumer's subscription[{}] error, group: {}", requestHeader.getExp(), + requestHeader.getConsumerGroup()); + response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED); + response.setRemark("parse the consumer's subscription failed"); + return response; + } + } + + int randomQ = random.nextInt(100); + int reviveQid; + if (requestHeader.isOrder()) { + reviveQid = KeyBuilder.POP_ORDER_REVIVE_QUEUE; + } else { + reviveQid = randomQ % this.brokerController.getBrokerConfig().getReviveQueueNum(); + } + + GetMessageResult getMessageResult = new GetMessageResult(); + + long restNum = 0; + boolean needRetry = randomQ % 5 == 0; + long popTime = System.currentTimeMillis(); + if (needRetry && !requestHeader.isOrder()) { + TopicConfig retryTopicConfig = + this.brokerController.getTopicConfigManager().selectTopicConfig(KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup())); + if (retryTopicConfig != null) { + for (int i = 0; i < retryTopicConfig.getReadQueueNums(); i++) { + int queueId = (randomQ + i) % retryTopicConfig.getReadQueueNums(); + restNum = popMsgFromQueue(true, getMessageResult, requestHeader, queueId, restNum, reviveQid, + channel, popTime, messageFilter, + startOffsetInfo, msgOffsetInfo, orderCountInfo); + } + } + } + if (requestHeader.getQueueId() < 0) { + // read all queue + for (int i = 0; i < topicConfig.getReadQueueNums(); i++) { + int queueId = (randomQ + i) % topicConfig.getReadQueueNums(); + restNum = popMsgFromQueue(false, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime, messageFilter, + startOffsetInfo, msgOffsetInfo, orderCountInfo); + } + } else { + int queueId = requestHeader.getQueueId(); + restNum = popMsgFromQueue(false, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, + popTime, messageFilter, + startOffsetInfo, msgOffsetInfo, orderCountInfo); + } + // if not full , fetch retry again + if (!needRetry && getMessageResult.getMessageMapedList().size() < requestHeader.getMaxMsgNums() && !requestHeader.isOrder()) { + TopicConfig retryTopicConfig = + this.brokerController.getTopicConfigManager().selectTopicConfig(KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup())); + if (retryTopicConfig != null) { + for (int i = 0; i < retryTopicConfig.getReadQueueNums(); i++) { + int queueId = (randomQ + i) % retryTopicConfig.getReadQueueNums(); + restNum = popMsgFromQueue(true, getMessageResult, requestHeader, queueId, restNum, reviveQid, + channel, popTime, messageFilter, + startOffsetInfo, msgOffsetInfo, orderCountInfo); + } + } + } + if (!getMessageResult.getMessageBufferList().isEmpty()) { + response.setCode(ResponseCode.SUCCESS); + getMessageResult.setStatus(GetMessageStatus.FOUND); + if (restNum > 0) { + // all queue pop can not notify specified queue pop, and vice versa + notifyMessageArriving(requestHeader.getTopic(), requestHeader.getConsumerGroup(), + requestHeader.getQueueId()); + } + } else { + int pollingResult = polling(channel, request, requestHeader); + if (POLLING_SUC == pollingResult) { + return null; + } else if (POLLING_FULL == pollingResult) { + response.setCode(ResponseCode.POLLING_FULL); + } else { + response.setCode(ResponseCode.POLLING_TIMEOUT); + } + getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE); + } + responseHeader.setInvisibleTime(requestHeader.getInvisibleTime()); + responseHeader.setPopTime(popTime); + responseHeader.setReviveQid(reviveQid); + responseHeader.setRestNum(restNum); + responseHeader.setStartOffsetInfo(startOffsetInfo.toString()); + responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString()); + if (requestHeader.isOrder() && orderCountInfo != null) { + responseHeader.setOrderCountInfo(orderCountInfo.toString()); + } + response.setRemark(getMessageResult.getStatus().name()); + switch (response.getCode()) { + case ResponseCode.SUCCESS: + if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) { + final long beginTimeMills = this.brokerController.getMessageStore().now(); + final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(), + requestHeader.getTopic(), requestHeader.getQueueId()); + this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(), + requestHeader.getTopic(), requestHeader.getQueueId(), + (int) (this.brokerController.getMessageStore().now() - beginTimeMills)); + response.setBody(r); + } else { + final GetMessageResult tmpGetMessageResult = getMessageResult; + try { + FileRegion fileRegion = + new ManyMessageTransfer(response.encodeHeader(getMessageResult.getBufferTotalSize()), + getMessageResult); + channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + tmpGetMessageResult.release(); + if (!future.isSuccess()) { + POP_LOGGER.error("Fail to transfer messages from page cache to {}", + channel.remoteAddress(), future.cause()); + } + } + }); + } catch (Throwable e) { + POP_LOGGER.error("Error occurred when transferring messages from page cache", e); + getMessageResult.release(); + } + + response = null; + } + break; + default: + assert false; + } + return response; + } + + private long popMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult, + PopMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid, + Channel channel, long popTime, + ExpressionMessageFilter messageFilter, StringBuilder startOffsetInfo, + StringBuilder msgOffsetInfo, StringBuilder orderCountInfo) { + String topic = isRetry ? KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), + requestHeader.getConsumerGroup()) : requestHeader.getTopic(); + String lockKey = + topic + PopAckConstants.SPLIT + requestHeader.getConsumerGroup() + PopAckConstants.SPLIT + queueId; + boolean isOrder = requestHeader.isOrder(); + long offset = getPopOffset(topic, requestHeader, queueId, false, lockKey); + if (!queueLockManager.tryLock(lockKey)) { + restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + return restNum; + } + offset = getPopOffset(topic, requestHeader, queueId, true, lockKey); + GetMessageResult getMessageTmpResult; + try { + if (isOrder && brokerController.getConsumerOrderInfoManager().checkBlock(topic, + requestHeader.getConsumerGroup(), queueId, requestHeader.getInvisibleTime())) { + return this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + } + + if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) { + restNum = + this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + return restNum; + } + getMessageTmpResult = this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup() + , topic, queueId, offset, + requestHeader.getMaxMsgNums() - getMessageResult.getMessageMapedList().size(), messageFilter); + // maybe store offset is not correct. + if (getMessageTmpResult == null + || GetMessageStatus.OFFSET_TOO_SMALL.equals(getMessageTmpResult.getStatus()) + || GetMessageStatus.OFFSET_OVERFLOW_BADLY.equals(getMessageTmpResult.getStatus()) + || GetMessageStatus.OFFSET_FOUND_NULL.equals(getMessageTmpResult.getStatus())) { + // commit offset, because the offset is not correct + // If offset in store is greater than cq offset, it will cause duplicate messages, + // because offset in PopBuffer is not committed. + POP_LOGGER.warn("Pop initial offset, because store is no correct, {}, {}->{}", + lockKey, offset, getMessageTmpResult != null ? getMessageTmpResult.getNextBeginOffset() : "null"); + offset = getMessageTmpResult != null ? getMessageTmpResult.getNextBeginOffset() : 0; + this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic, + queueId, offset); + getMessageTmpResult = + this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), topic, + queueId, offset, + requestHeader.getMaxMsgNums() - getMessageResult.getMessageMapedList().size(), messageFilter); + } + + restNum = getMessageTmpResult.getMaxOffset() - getMessageTmpResult.getNextBeginOffset() + restNum; + if (!getMessageTmpResult.getMessageMapedList().isEmpty()) { + this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageTmpResult.getMessageCount()); + this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), topic, + getMessageTmpResult.getMessageCount()); + this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), topic, + getMessageTmpResult.getBufferTotalSize()); + + if (isOrder) { + int count = brokerController.getConsumerOrderInfoManager().update(topic, + requestHeader.getConsumerGroup(), + queueId, getMessageTmpResult.getMessageQueueOffset()); + this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), + requestHeader.getConsumerGroup(), topic, queueId, offset); + ExtraInfoUtil.buildOrderCountInfo(orderCountInfo, isRetry, queueId, count); + } else { + appendCheckPoint(requestHeader, topic, reviveQid, queueId, offset, getMessageTmpResult, popTime); + } + ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, isRetry, queueId, offset); + ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, isRetry, queueId, + getMessageTmpResult.getMessageQueueOffset()); + } else if ((GetMessageStatus.NO_MATCHED_MESSAGE.equals(getMessageTmpResult.getStatus()) + || GetMessageStatus.OFFSET_FOUND_NULL.equals(getMessageTmpResult.getStatus()) + || GetMessageStatus.MESSAGE_WAS_REMOVING.equals(getMessageTmpResult.getStatus()) + || GetMessageStatus.NO_MATCHED_LOGIC_QUEUE.equals(getMessageTmpResult.getStatus())) + && getMessageTmpResult.getNextBeginOffset() > -1) { + popBufferMergeService.addCkMock(requestHeader.getConsumerGroup(), topic, queueId, offset, + requestHeader.getInvisibleTime(), popTime, reviveQid, getMessageTmpResult.getNextBeginOffset()); + } + } finally { + queueLockManager.unLock(lockKey); + } + if (getMessageTmpResult != null) { + for (SelectMappedBufferResult mapedBuffer : getMessageTmpResult.getMessageMapedList()) { + getMessageResult.addMessage(mapedBuffer); + } + } + return restNum; + } + + private long getPopOffset(String topic, PopMessageRequestHeader requestHeader, int queueId, boolean init, + String lockKey) { + long offset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(), + topic, queueId); + if (offset < 0) { + if (ConsumeInitMode.MIN == requestHeader.getInitMode()) { + offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId); + } else { + // pop last one,then commit offset. + offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - 1; + // max & no consumer offset + if (offset < 0) { + offset = 0; + } + if (init) { + this.brokerController.getConsumerOffsetManager().commitOffset("getPopOffset", + requestHeader.getConsumerGroup(), topic, + queueId, offset); + } + } + } + long bufferOffset = this.popBufferMergeService.getLatestOffset(lockKey); + if (bufferOffset < 0) { + return offset; + } else { + return bufferOffset > offset ? bufferOffset : offset; + } + } + + /** + * @param channel + * @param remotingCommand + * @param requestHeader + * @return + */ + private int polling(final Channel channel, RemotingCommand remotingCommand, + final PopMessageRequestHeader requestHeader) { + if (requestHeader.getPollTime() <= 0 || this.popLongPollingService.isStopped()) { + return NOT_POLLING; + } + ConcurrentHashMap cids = topicCidMap.get(requestHeader.getTopic()); + if (cids == null) { + cids = new ConcurrentHashMap<>(); + ConcurrentHashMap old = topicCidMap.putIfAbsent(requestHeader.getTopic(), cids); + if (old != null) { + cids = old; + } + } + cids.putIfAbsent(requestHeader.getConsumerGroup(), Byte.MIN_VALUE); + long expired = requestHeader.getBornTime() + requestHeader.getPollTime(); + final PopRequest request = new PopRequest(remotingCommand, channel, expired); + boolean isFull = totalPollingNum.get() >= this.brokerController.getBrokerConfig().getMaxPopPollingSize(); + if (isFull) { + POP_LOGGER.info("polling {}, result POLLING_FULL, total:{}", remotingCommand, totalPollingNum.get()); + return POLLING_FULL; + } + boolean isTimeout = request.isTimeout(); + if (isTimeout) { + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("polling {}, result POLLING_TIMEOUT", remotingCommand); + } + return POLLING_TIMEOUT; + } + String key = KeyBuilder.buildPollingKey(requestHeader.getTopic(), requestHeader.getConsumerGroup(), + requestHeader.getQueueId()); + ConcurrentSkipListSet queue = pollingMap.get(key); + if (queue == null) { + queue = new ConcurrentSkipListSet<>(PopRequest.COMPARATOR); + ConcurrentSkipListSet old = pollingMap.putIfAbsent(key, queue); + if (old != null) { + queue = old; + } + } else { + // check size + int size = queue.size(); + if (size > brokerController.getBrokerConfig().getPopPollingSize()) { + POP_LOGGER.info("polling {}, result POLLING_FULL, singleSize:{}", remotingCommand, size); + return POLLING_FULL; + } + } + if (queue.add(request)) { + totalPollingNum.incrementAndGet(); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("polling {}, result POLLING_SUC", remotingCommand); + } + return POLLING_SUC; + } else { + POP_LOGGER.info("polling {}, result POLLING_FULL, add fail, {}", request, queue); + return POLLING_FULL; + } + } + + public final MessageExtBrokerInner buildCkMsg(final PopCheckPoint ck, final int reviveQid) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + + msgInner.setTopic(reviveTopic); + msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); + msgInner.setQueueId(reviveQid); + msgInner.setTags(PopAckConstants.CK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); + msgInner.setBornHost(this.brokerController.getStoreHost()); + msgInner.setStoreHost(this.brokerController.getStoreHost()); + MsgUtil.setMessageDeliverTime(this.brokerController, msgInner, ck.getReviveTime() - PopAckConstants.ackTimeInterval); + msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, genCkUniqueId(ck)); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + + return msgInner; + } + + private void appendCheckPoint(final PopMessageRequestHeader requestHeader, + final String topic, final int reviveQid, final int queueId, final long offset, + final GetMessageResult getMessageTmpResult, final long popTime) { + // add check point msg to revive log + final PopCheckPoint ck = new PopCheckPoint(); + ck.setBitMap(0); + ck.setNum((byte) getMessageTmpResult.getMessageMapedList().size()); + ck.setPopTime(popTime); + ck.setInvisibleTime(requestHeader.getInvisibleTime()); + ck.getStartOffset(offset); + ck.setCId(requestHeader.getConsumerGroup()); + ck.setTopic(topic); + ck.setQueueId((byte) queueId); + for (Long msgQueueOffset : getMessageTmpResult.getMessageQueueOffset()) { + ck.addDiff((int) (msgQueueOffset - offset)); + } + + final boolean addBufferSuc = this.popBufferMergeService.addCk( + ck, reviveQid, -1, getMessageTmpResult.getNextBeginOffset() + ); + + if (addBufferSuc) { + return; + } + + this.popBufferMergeService.addCkJustOffset( + ck, reviveQid, -1, getMessageTmpResult.getNextBeginOffset() + ); + } + + private byte[] readGetMessageResult(final GetMessageResult getMessageResult, final String group, + final String topic, final int queueId) { + final ByteBuffer byteBuffer = ByteBuffer.allocate(getMessageResult.getBufferTotalSize()); + + long storeTimestamp = 0; + try { + List messageBufferList = getMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + + byteBuffer.put(bb); + storeTimestamp = bb.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION); + } + } finally { + getMessageResult.release(); + } + + this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, + this.brokerController.getMessageStore().now() - storeTimestamp); + return byteBuffer.array(); + } + + public class PopLongPollingService extends ServiceThread { + + private long lastCleanTime = 0; + + @Override + public String getServiceName() { + return "PopLongPollingService"; + } + + private void cleanUnusedResource() { + try { + { + Iterator>> topicCidMapIter = topicCidMap.entrySet().iterator(); + while (topicCidMapIter.hasNext()) { + Entry> entry = topicCidMapIter.next(); + String topic = entry.getKey(); + if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) { + POP_LOGGER.info("remove not exit topic {} in topicCidMap!", topic); + topicCidMapIter.remove(); + continue; + } + Iterator> cidMapIter = entry.getValue().entrySet().iterator(); + while (cidMapIter.hasNext()) { + Entry cidEntry = cidMapIter.next(); + String cid = cidEntry.getKey(); + if (!brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().containsKey(cid)) { + POP_LOGGER.info("remove not exit sub {} of topic {} in topicCidMap!", cid, topic); + cidMapIter.remove(); + } + } + } + } + + { + Iterator>> pollingMapIter = pollingMap.entrySet().iterator(); + while (pollingMapIter.hasNext()) { + Entry> entry = pollingMapIter.next(); + if (entry.getKey() == null) { + continue; + } + String[] keyArray = entry.getKey().split(PopAckConstants.SPLIT); + if (keyArray == null || keyArray.length != 3) { + continue; + } + String topic = keyArray[0]; + String cid = keyArray[1]; + if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) { + POP_LOGGER.info("remove not exit topic {} in pollingMap!", topic); + pollingMapIter.remove(); + continue; + } + if (!brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().containsKey(cid)) { + POP_LOGGER.info("remove not exit sub {} of topic {} in pollingMap!", cid, topic); + pollingMapIter.remove(); + continue; + } + } + } + } catch (Throwable e) { + POP_LOGGER.error("cleanUnusedResource", e); + } + + lastCleanTime = System.currentTimeMillis(); + } + + @Override + public void run() { + int i = 0; + while (!this.stopped) { + try { + this.waitForRunning(20); + i++; + if (pollingMap.isEmpty()) { + continue; + } + long tmpTotalPollingNum = 0; + Iterator>> pollingMapIterator = pollingMap.entrySet().iterator(); + while (pollingMapIterator.hasNext()) { + Entry> entry = pollingMapIterator.next(); + String key = entry.getKey(); + ConcurrentSkipListSet popQ = entry.getValue(); + if (popQ == null) { + continue; + } + PopRequest first; + do { + first = popQ.pollFirst(); + if (first == null) { + break; + } + if (!first.isTimeout()) { + if (popQ.add(first)) { + break; + } else { + POP_LOGGER.info("polling, add fail again: {}", first); + } + } + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("timeout , wakeUp polling : {}", first); + } + totalPollingNum.decrementAndGet(); + wakeUp(first); + } while (true); + if (i >= 100) { + long tmpPollingNum = popQ.size(); + tmpTotalPollingNum = tmpTotalPollingNum + tmpPollingNum; + if (tmpPollingNum > 100) { + POP_LOGGER.info("polling queue {} , size={} ", key, tmpPollingNum); + } + } + } + + if (i >= 100) { + POP_LOGGER.info("pollingMapSize={},tmpTotalSize={},atomicTotalSize={},diffSize={}", + pollingMap.size(), tmpTotalPollingNum, totalPollingNum.get(), + Math.abs(totalPollingNum.get() - tmpTotalPollingNum)); + totalPollingNum.set(tmpTotalPollingNum); + i = 0; + } + + // clean unused + if (lastCleanTime == 0 || System.currentTimeMillis() - lastCleanTime > 5 * 60 * 1000) { + cleanUnusedResource(); + } + } catch (Throwable e) { + POP_LOGGER.error("checkPolling error", e); + } + } + // clean all; + try { + Iterator>> pollingMapIterator = pollingMap.entrySet().iterator(); + while (pollingMapIterator.hasNext()) { + Entry> entry = pollingMapIterator.next(); + ConcurrentSkipListSet popQ = entry.getValue(); + PopRequest first; + while ((first = popQ.pollFirst()) != null) { + wakeUp(first); + } + } + } catch (Throwable e) { + } + } + } + + static class TimedLock { + private final AtomicBoolean lock; + private volatile long lockTime; + + public TimedLock() { + this.lock = new AtomicBoolean(true); + this.lockTime = System.currentTimeMillis(); + } + + public boolean tryLock() { + boolean ret = lock.compareAndSet(true, false); + if (ret) { + this.lockTime = System.currentTimeMillis(); + return true; + } else { + return false; + } + } + + public void unLock() { + lock.set(true); + } + + public boolean isLock() { + return !lock.get(); + } + + public long getLockTime() { + return lockTime; + } + } + + public static class QueueLockManager extends ServiceThread { + private ConcurrentHashMap expiredLocalCache = new ConcurrentHashMap<>(100000); + + public boolean tryLock(String key) { + TimedLock timedLock = expiredLocalCache.get(key); + + if (timedLock == null) { + TimedLock old = expiredLocalCache.putIfAbsent(key, new TimedLock()); + if (old != null) { + return false; + } else { + timedLock = expiredLocalCache.get(key); + } + } + + if (timedLock == null) { + return false; + } + + return timedLock.tryLock(); + } + + /** + * is not thread safe, may cause duplicate lock + * + * @param usedExpireMillis + * @return + */ + public int cleanUnusedLock(final long usedExpireMillis) { + Iterator> iterator = expiredLocalCache.entrySet().iterator(); + + int total = 0; + while (iterator.hasNext()) { + Entry entry = iterator.next(); + + if (System.currentTimeMillis() - entry.getValue().getLockTime() > usedExpireMillis) { + iterator.remove(); + POP_LOGGER.info("Remove unused queue lock: {}, {}, {}", entry.getKey(), + entry.getValue().getLockTime(), + entry.getValue().isLock()); + } + + total++; + } + + return total; + } + + public void unLock(String key) { + TimedLock timedLock = expiredLocalCache.get(key); + if (timedLock != null) { + timedLock.unLock(); + } + } + + @Override + public String getServiceName() { + return "QueueLockManager"; + } + + @Override + public void run() { + while (!isStopped()) { + try { + this.waitForRunning(60000); + int count = cleanUnusedLock(60000); + POP_LOGGER.info("QueueLockSize={}", count); + } catch (Exception e) { + PopMessageProcessor.POP_LOGGER.error("QueueLockManager run error", e); + } + } + } + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java new file mode 100644 index 0000000000000000000000000000000000000000..b9c309e189878ac942bc2ee24fb7a655b6c4fe26 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -0,0 +1,461 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import com.alibaba.fastjson.JSON; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.util.MsgUtil; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.PopAckConstants; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.TopicFilterType; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.pop.AckMsg; +import org.apache.rocketmq.store.pop.PopCheckPoint; + +public class PopReviveService extends ServiceThread { + private static final InternalLogger POP_LOGGER = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME); + + private int queueId; + private BrokerController brokerController; + private String reviveTopic; + private static volatile boolean isMaster = false; + + public PopReviveService(int queueId, BrokerController brokerController, String reviveTopic) { + super(); + this.queueId = queueId; + this.brokerController = brokerController; + this.reviveTopic = reviveTopic; + } + + @Override + public String getServiceName() { + return "PopReviveService_" + this.queueId; + } + + private boolean checkMaster() { + return brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE; + } + + private boolean checkAndSetMaster() { + isMaster = checkMaster(); + return isMaster; + } + + private void reviveRetry(PopCheckPoint popCheckPoint, MessageExt messageExt) throws Exception { + if (!checkAndSetMaster()) { + POP_LOGGER.info("slave skip retry , revive topic={}, reviveQueueId={}", reviveTopic, queueId); + return; + } + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + if (!popCheckPoint.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + msgInner.setTopic(KeyBuilder.buildPopRetryTopic(popCheckPoint.getTopic(), popCheckPoint.getCId())); + } else { + msgInner.setTopic(popCheckPoint.getTopic()); + } + msgInner.setBody(messageExt.getBody()); + msgInner.setQueueId(0); + if (messageExt.getTags() != null) { + msgInner.setTags(messageExt.getTags()); + } else { + MessageAccessor.setProperties(msgInner, new HashMap()); + } + msgInner.setBornTimestamp(messageExt.getBornTimestamp()); + msgInner.setBornHost(brokerController.getStoreHost()); + msgInner.setStoreHost(brokerController.getStoreHost()); + msgInner.setReconsumeTimes(messageExt.getReconsumeTimes() + 1); + msgInner.getProperties().putAll(messageExt.getProperties()); + if (messageExt.getReconsumeTimes() == 0 || msgInner.getProperties().get(MessageConst.PROPERTY_FIRST_POP_TIME) == null) { + msgInner.getProperties().put(MessageConst.PROPERTY_FIRST_POP_TIME, String.valueOf(popCheckPoint.getPopTime())); + } + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + addRetryTopicIfNoExit(msgInner.getTopic()); + PutMessageResult putMessageResult = brokerController.getMessageStore().putMessage(msgInner); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("reviveQueueId={},retry msg , ck={}, msg queueId {}, offset {}, reviveDelay={}, result is {} ", + queueId, popCheckPoint, messageExt.getQueueId(), messageExt.getQueueOffset(), + (System.currentTimeMillis() - popCheckPoint.getReviveTime()) / 1000, putMessageResult); + } + if (putMessageResult.getAppendMessageResult() == null || putMessageResult.getAppendMessageResult().getStatus() != AppendMessageStatus.PUT_OK) { + throw new Exception("reviveQueueId=" + queueId + ",revive error ,msg is :" + msgInner); + } + this.brokerController.getBrokerStatsManager().incBrokerPutNums(1); + this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic()); + this.brokerController.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes()); + if (brokerController.getPopMessageProcessor() != null) { + brokerController.getPopMessageProcessor().notifyMessageArriving( + KeyBuilder.parseNormalTopic(popCheckPoint.getTopic(), popCheckPoint.getCId()), + popCheckPoint.getCId(), + -1 + ); + } + } + + private boolean addRetryTopicIfNoExit(String topic) { + TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (topicConfig != null) { + return true; + } + topicConfig = new TopicConfig(topic); + topicConfig.setReadQueueNums(PopAckConstants.retryQueueNum); + topicConfig.setWriteQueueNums(PopAckConstants.retryQueueNum); + topicConfig.setTopicFilterType(TopicFilterType.SINGLE_TAG); + topicConfig.setPerm(6); + topicConfig.setTopicSysFlag(0); + brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); + return true; + } + + private List getReviveMessage(long offset, int queueId) { + PullResult pullResult = getMessage(PopAckConstants.REVIVE_GROUP, reviveTopic, queueId, offset, 32); + if (pullResult == null) { + return null; + } + if (reachTail(pullResult, offset)) { + POP_LOGGER.info("reviveQueueId={}, reach tail,offset {}", queueId, offset); + } else if (pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL || pullResult.getPullStatus() == PullStatus.NO_MATCHED_MSG) { + POP_LOGGER.error("reviveQueueId={}, OFFSET_ILLEGAL {}, result is {}", queueId, offset, pullResult); + if (!checkAndSetMaster()) { + POP_LOGGER.info("slave skip offset correct topic={}, reviveQueueId={}", reviveTopic, queueId); + return null; + } + brokerController.getConsumerOffsetManager().commitOffset(PopAckConstants.LOCAL_HOST, PopAckConstants.REVIVE_GROUP, reviveTopic, queueId, pullResult.getNextBeginOffset() - 1); + } + return pullResult.getMsgFoundList(); + } + + private boolean reachTail(PullResult pullResult, long offset) { + return pullResult.getPullStatus() == PullStatus.NO_NEW_MSG + || (pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL && offset == pullResult.getMaxOffset()); + } + + private MessageExt getBizMessage(String topic, long offset, int queueId) { + final GetMessageResult getMessageTmpResult = brokerController.getMessageStore().getMessage(PopAckConstants.REVIVE_GROUP, topic, queueId, offset, 1, null); + List list = decodeMsgList(getMessageTmpResult); + if (list == null || list.isEmpty()) { + POP_LOGGER.warn("can not get msg , topic {}, offset {}, queueId {}, result is {}", topic, offset, queueId, getMessageTmpResult); + return null; + } else { + return list.get(0); + } + } + + public PullResult getMessage(String group, String topic, int queueId, long offset, int nums) { + GetMessageResult getMessageResult = brokerController.getMessageStore().getMessage(group, topic, queueId, offset, nums, null); + + if (getMessageResult != null) { + PullStatus pullStatus = PullStatus.NO_NEW_MSG; + List foundList = null; + switch (getMessageResult.getStatus()) { + case FOUND: + pullStatus = PullStatus.FOUND; + foundList = decodeMsgList(getMessageResult); + brokerController.getBrokerStatsManager().incGroupGetNums(group, topic, getMessageResult.getMessageCount()); + brokerController.getBrokerStatsManager().incGroupGetSize(group, topic, getMessageResult.getBufferTotalSize()); + brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount()); + brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, + brokerController.getMessageStore().now() - foundList.get(foundList.size() - 1).getStoreTimestamp()); + break; + case NO_MATCHED_MESSAGE: + pullStatus = PullStatus.NO_MATCHED_MSG; + POP_LOGGER.warn("no matched message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + getMessageResult.getStatus(), topic, group, offset); + break; + case NO_MESSAGE_IN_QUEUE: + pullStatus = PullStatus.NO_NEW_MSG; + POP_LOGGER.warn("no new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + getMessageResult.getStatus(), topic, group, offset); + break; + case MESSAGE_WAS_REMOVING: + case NO_MATCHED_LOGIC_QUEUE: + case OFFSET_FOUND_NULL: + case OFFSET_OVERFLOW_BADLY: + case OFFSET_OVERFLOW_ONE: + case OFFSET_TOO_SMALL: + pullStatus = PullStatus.OFFSET_ILLEGAL; + POP_LOGGER.warn("offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", + getMessageResult.getStatus(), topic, group, offset); + break; + default: + assert false; + break; + } + + return new PullResult(pullStatus, getMessageResult.getNextBeginOffset(), getMessageResult.getMinOffset(), + getMessageResult.getMaxOffset(), foundList); + + } else { + POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}", topic, group, offset); + return null; + } + } + + private List decodeMsgList(GetMessageResult getMessageResult) { + List foundList = new ArrayList<>(); + try { + List messageBufferList = getMessageResult.getMessageBufferList(); + if (messageBufferList != null) { + for (int i = 0; i < messageBufferList.size(); i++) { + ByteBuffer bb = messageBufferList.get(i); + if (bb == null) { + POP_LOGGER.error("bb is null {}", getMessageResult); + continue; + } + MessageExt msgExt = MessageDecoder.decode(bb); + if (msgExt == null) { + POP_LOGGER.error("decode msgExt is null {}", getMessageResult); + continue; + } + // use CQ offset, not offset in Message + msgExt.setQueueOffset(getMessageResult.getMessageQueueOffset().get(i)); + foundList.add(msgExt); + } + } + } finally { + getMessageResult.release(); + } + + return foundList; + } + + private void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { + HashMap map = consumeReviveObj.map; + long startScanTime = System.currentTimeMillis(); + long endTime = 0; + long oldOffset = brokerController.getConsumerOffsetManager().queryOffset(PopAckConstants.REVIVE_GROUP, reviveTopic, queueId); + consumeReviveObj.oldOffset = oldOffset; + POP_LOGGER.info("reviveQueueId={}, old offset is {} ", queueId, oldOffset); + long offset = oldOffset + 1; + long firstRt = 0; + // offset self amend + while (true) { + if (!checkAndSetMaster()) { + POP_LOGGER.info("slave skip scan , revive topic={}, reviveQueueId={}", reviveTopic, queueId); + break; + } + List messageExts = getReviveMessage(offset, queueId); + if (messageExts == null || messageExts.isEmpty()) { + break; + } + if (System.currentTimeMillis() - startScanTime > brokerController.getBrokerConfig().getReviveScanTime()) { + POP_LOGGER.info("reviveQueueId={}, scan timeout ", queueId); + break; + } + for (MessageExt messageExt : messageExts) { + if (PopAckConstants.CK_TAG.equals(messageExt.getTags())) { + String raw = new String(messageExt.getBody(), DataConverter.charset); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("reviveQueueId={},find ck, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); + } + PopCheckPoint point = JSON.parseObject(raw, PopCheckPoint.class); + if (point.getTopic() == null || point.getCId() == null) { + continue; + } + map.put(point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime(), point); + point.setReviveOffset(messageExt.getQueueOffset()); + if (firstRt == 0) { + firstRt = point.getReviveTime(); + } + } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) { + String raw = new String(messageExt.getBody(), DataConverter.charset); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("reviveQueueId={},find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); + } + AckMsg ackMsg = JSON.parseObject(raw, AckMsg.class); + PopCheckPoint point = map.get(ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime()); + if (point == null) { + continue; + } + int indexOfAck = point.indexOfAck(ackMsg.getAckOffset()); + if (indexOfAck > -1) { + point.setBitMap(DataConverter.setBit(point.getBitMap(), indexOfAck, true)); + } else { + POP_LOGGER.error("invalid ack index, {}, {}", ackMsg, point); + } + } + long deliverTime = MsgUtil.getMessageDeliverTime(this.brokerController, messageExt); + if (deliverTime > endTime) { + endTime = deliverTime; + } + } + offset = offset + messageExts.size(); + } + consumeReviveObj.endTime = endTime; + } + + private void mergeAndRevive(ConsumeReviveObj consumeReviveObj) throws Throwable { + ArrayList sortList = consumeReviveObj.genSortList(); + POP_LOGGER.info("reviveQueueId={},ck listSize={}", queueId, sortList.size()); + if (sortList.size() != 0) { + POP_LOGGER.info("reviveQueueId={}, 1st ck, startOffset={}, reviveOffset={} ; last ck, startOffset={}, reviveOffset={}", queueId, sortList.get(0).getStartOffset(), + sortList.get(0).getReviveOffset(), sortList.get(sortList.size() - 1).getStartOffset(), sortList.get(sortList.size() - 1).getReviveOffset()); + } + long newOffset = consumeReviveObj.oldOffset; + for (PopCheckPoint popCheckPoint : sortList) { + if (!checkAndSetMaster()) { + POP_LOGGER.info("slave skip ck process , revive topic={}, reviveQueueId={}", reviveTopic, queueId); + break; + } + if (consumeReviveObj.endTime - popCheckPoint.getReviveTime() <= (PopAckConstants.ackTimeInterval + PopAckConstants.SECOND)) { + break; + } + + // check normal topic, skip ck , if normal topic is not exist + String normalTopic = KeyBuilder.parseNormalTopic(popCheckPoint.getTopic(), popCheckPoint.getCId()); + if (brokerController.getTopicConfigManager().selectTopicConfig(normalTopic) == null) { + POP_LOGGER.warn("reviveQueueId={},can not get normal topic {} , then continue ", queueId, popCheckPoint.getTopic()); + newOffset = popCheckPoint.getReviveOffset(); + continue; + } + if (null == brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(popCheckPoint.getCId())) { + POP_LOGGER.warn("reviveQueueId={},can not get cid {} , then continue ", queueId, popCheckPoint.getCId()); + newOffset = popCheckPoint.getReviveOffset(); + continue; + } + + reviveMsgFromCk(popCheckPoint); + + newOffset = popCheckPoint.getReviveOffset(); + } + if (newOffset > consumeReviveObj.oldOffset) { + if (!checkAndSetMaster()) { + POP_LOGGER.info("slave skip commit, revive topic={}, reviveQueueId={}", reviveTopic, queueId); + return; + } + brokerController.getConsumerOffsetManager().commitOffset(PopAckConstants.LOCAL_HOST, PopAckConstants.REVIVE_GROUP, reviveTopic, queueId, newOffset); + } + consumeReviveObj.newOffset = newOffset; + } + + private void reviveMsgFromCk(PopCheckPoint popCheckPoint) throws Throwable { + for (int j = 0; j < popCheckPoint.getNum(); j++) { + if (DataConverter.getBit(popCheckPoint.getBitMap(), j)) { + continue; + } + + // retry msg + long msgOffset = popCheckPoint.ackOffsetByIndex((byte) j); + MessageExt messageExt = getBizMessage(popCheckPoint.getTopic(), msgOffset, popCheckPoint.getQueueId()); + if (messageExt == null) { + POP_LOGGER.warn("reviveQueueId={},can not get biz msg topic is {}, offset is {} , then continue ", + queueId, popCheckPoint.getTopic(), msgOffset); + continue; + } + //skip ck from last epoch + if (popCheckPoint.getPopTime() < messageExt.getStoreTimestamp()) { + POP_LOGGER.warn("reviveQueueId={},skip ck from last epoch {}", queueId, popCheckPoint); + continue; + } + reviveRetry(popCheckPoint, messageExt); + } + } + + @Override + public void run() { + int slow = 1; + while (!this.isStopped()) { + try { + if (System.currentTimeMillis() < brokerController.getShouldStartTime()) { + POP_LOGGER.info("PopReviveService Ready to run after {}", brokerController.getShouldStartTime()); + this.waitForRunning(1000); + continue; + } + this.waitForRunning(brokerController.getBrokerConfig().getReviveInterval()); + if (!checkAndSetMaster()) { + POP_LOGGER.info("slave skip start revive topic={}, reviveQueueId={}", reviveTopic, queueId); + continue; + } + + POP_LOGGER.info("start revive topic={}, reviveQueueId={}", reviveTopic, queueId); + ConsumeReviveObj consumeReviveObj = new ConsumeReviveObj(); + consumeReviveMessage(consumeReviveObj); + + if (!checkAndSetMaster()) { + POP_LOGGER.info("slave skip scan , revive topic={}, reviveQueueId={}", reviveTopic, queueId); + continue; + } + + mergeAndRevive(consumeReviveObj); + + ArrayList sortList = consumeReviveObj.sortList; + long delay = 0; + if (sortList != null && !sortList.isEmpty()) { + delay = (System.currentTimeMillis() - sortList.get(0).getReviveTime()) / 1000; + slow = 1; + } + + POP_LOGGER.info("reviveQueueId={},revive finish,old offset is {}, new offset is {}, ckDelay={} ", + queueId, consumeReviveObj.oldOffset, consumeReviveObj.newOffset, delay); + + if (sortList == null || sortList.isEmpty()) { + POP_LOGGER.info("reviveQueueId={},has no new msg ,take a rest {}", queueId, slow); + this.waitForRunning(slow * brokerController.getBrokerConfig().getReviveInterval()); + if (slow < brokerController.getBrokerConfig().getReviveMaxSlow()) { + slow++; + } + } + + } catch (Throwable e) { + POP_LOGGER.error("reviveQueueId=" + queueId + ",revive error", e); + } + } + } + + static class ConsumeReviveObj { + HashMap map = new HashMap<>(); + ArrayList sortList; + long oldOffset; + long endTime; + long newOffset; + + ArrayList genSortList() { + if (sortList != null) { + return sortList; + } + sortList = new ArrayList<>(map.values()); + Collections.sort(sortList, new Comparator() { + @Override + public int compare(PopCheckPoint o1, PopCheckPoint o2) { + return (int) (o1.getReviveOffset() - o2.getReviveOffset()); + } + }); + return sortList; + } + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java new file mode 100644 index 0000000000000000000000000000000000000000..fdc320de87f3062398bb5b69692e821fbed951d2 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.ChannelHandlerContext; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.loadbalance.AssignmentManager; +import org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager; +import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.message.MessageQueueAssignment; +import org.apache.rocketmq.common.message.MessageRequestMode; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.QueryAssignmentRequestBody; +import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody; +import org.apache.rocketmq.common.protocol.body.SetMessageRequestModeRequestBody; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class QueryAssignmentProcessor implements NettyRequestProcessor { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + + private final BrokerController brokerController; + + private final ConcurrentHashMap name2LoadStrategy = new ConcurrentHashMap(); + + private MessageRequestModeManager messageRequestModeManager; + + public QueryAssignmentProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + + //register strategy + //NOTE: init with broker's log instead of init with ClientLogger.getLog(); + AllocateMessageQueueAveragely allocateMessageQueueAveragely = new AllocateMessageQueueAveragely(log); + name2LoadStrategy.put(allocateMessageQueueAveragely.getName(), allocateMessageQueueAveragely); + AllocateMessageQueueAveragelyByCircle allocateMessageQueueAveragelyByCircle = new AllocateMessageQueueAveragelyByCircle(log); + name2LoadStrategy.put(allocateMessageQueueAveragelyByCircle.getName(), allocateMessageQueueAveragelyByCircle); + + this.messageRequestModeManager = new MessageRequestModeManager(brokerController); + this.messageRequestModeManager.load(); + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + switch (request.getCode()) { + case RequestCode.QUERY_ASSIGNMENT: + return this.queryAssignment(ctx, request); + case RequestCode.SET_MESSAGE_REQUEST_MODE: + return this.setMessageRequestMode(ctx, request); + default: + break; + } + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } + + /** + * + */ + private RemotingCommand queryAssignment(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final QueryAssignmentRequestBody requestBody = QueryAssignmentRequestBody.decode(request.getBody(), QueryAssignmentRequestBody.class); + final String topic = requestBody.getTopic(); + final String consumerGroup = requestBody.getConsumerGroup(); + final String clientId = requestBody.getClientId(); + final MessageModel messageModel = requestBody.getMessageModel(); + final String strategyName = requestBody.getStrategyName(); + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final QueryAssignmentResponseBody responseBody = new QueryAssignmentResponseBody(); + + SetMessageRequestModeRequestBody setMessageRequestModeRequestBody = this.messageRequestModeManager.getMessageRequestMode(topic, consumerGroup); + + if (setMessageRequestModeRequestBody == null) { + setMessageRequestModeRequestBody = new SetMessageRequestModeRequestBody(); + setMessageRequestModeRequestBody.setTopic(topic); + setMessageRequestModeRequestBody.setConsumerGroup(consumerGroup); + + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + // retry topic must be pull mode + setMessageRequestModeRequestBody.setMode(MessageRequestMode.PULL); + } else { + setMessageRequestModeRequestBody.setMode(brokerController.getBrokerConfig().getDefaultMessageRequestMode()); + } + + if (setMessageRequestModeRequestBody.getMode() == MessageRequestMode.POP) { + setMessageRequestModeRequestBody.setPopShareQueueNum(brokerController.getBrokerConfig().getDefaultPopShareQueueNum()); + } + } + + Set messageQueues = doLoadBalance(topic, consumerGroup, clientId, messageModel, strategyName, setMessageRequestModeRequestBody, ctx); + + Set assignments = null; + if (messageQueues != null) { + assignments = new HashSet(); + for (MessageQueue messageQueue : messageQueues) { + MessageQueueAssignment messageQueueAssignment = new MessageQueueAssignment(); + messageQueueAssignment.setMessageQueue(messageQueue); + if (setMessageRequestModeRequestBody != null) { + messageQueueAssignment.setMode(setMessageRequestModeRequestBody.getMode()); + } + assignments.add(messageQueueAssignment); + } + } + + responseBody.setMessageQueueAssignments(assignments); + response.setBody(responseBody.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + /** + * Returns empty set means the client should clear all load assigned to it before, null means invalid result and the + * client should skip the update logic + * + * @param topic + * @param consumerGroup + * @param clientId + * @param messageModel + * @param strategyName + * @return the MessageQueues assigned to this client + */ + private Set doLoadBalance(final String topic, final String consumerGroup, final String clientId, + final MessageModel messageModel, final String strategyName, + SetMessageRequestModeRequestBody setMessageRequestModeRequestBody, final ChannelHandlerContext ctx) { + Set assignedQueueSet = null; + AssignmentManager assignmentManager = brokerController.getAssignmentManager(); + + switch (messageModel) { + case BROADCASTING: { + assignedQueueSet = assignmentManager.getTopicSubscribeInfo(topic); + if (assignedQueueSet == null) { + log.warn("QueryLoad: no assignment for group[{}], the topic[{}] does not exist.", consumerGroup, topic); + } + break; + } + case CLUSTERING: { + Set mqSet = assignmentManager.getTopicSubscribeInfo(topic); + if (null == mqSet) { + if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + log.warn("QueryLoad: no assignment for group[{}], the topic[{}] does not exist.", consumerGroup, topic); + } + return null; + } + + if (!brokerController.getBrokerConfig().isServerLoadBalancerEnabled()) { + return mqSet; + } + + List cidAll = null; + ConsumerGroupInfo consumerGroupInfo = this.brokerController.getConsumerManager().getConsumerGroupInfo(consumerGroup); + if (consumerGroupInfo != null) { + cidAll = consumerGroupInfo.getAllClientId(); + } + if (null == cidAll) { + log.warn("QueryLoad: no assignment for group[{}] topic[{}], get consumer id list failed", consumerGroup, topic); + return null; + } + + List mqAll = new ArrayList(); + mqAll.addAll(mqSet); + Collections.sort(mqAll); + Collections.sort(cidAll); + List allocateResult = null; + + try { + AllocateMessageQueueStrategy allocateMessageQueueStrategy = name2LoadStrategy.get(strategyName); + if (null == allocateMessageQueueStrategy) { + log.warn("QueryLoad: unsupported strategy [{}], {}", consumerGroup, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + return null; + } + + if (setMessageRequestModeRequestBody != null && setMessageRequestModeRequestBody.getMode() == MessageRequestMode.POP) { + if (setMessageRequestModeRequestBody.getPopShareQueueNum() <= 0) { + //each client pop all messagequeue + allocateResult = new ArrayList<>(mqAll.size()); + for (MessageQueue mq : mqAll) { + //must create new MessageQueue in case of change cache in AssignmentManager + MessageQueue newMq = new MessageQueue(mq.getTopic(), mq.getBrokerName(), -1); + allocateResult.add(newMq); + } + + } else { + if (cidAll.size() <= mqAll.size()) { + //consumer working in pop mode could share the MessageQueues assigned to the N (N = popWorkGroupSize) consumer following it in the cid list + allocateResult = allocateMessageQueueStrategy.allocate(consumerGroup, clientId, mqAll, cidAll); + int index = cidAll.indexOf(clientId); + if (index >= 0) { + for (int i = 1; i <= setMessageRequestModeRequestBody.getPopShareQueueNum(); i++) { + index++; + index = index % cidAll.size(); + List tmp = allocateMessageQueueStrategy.allocate(consumerGroup, cidAll.get(index), mqAll, cidAll); + allocateResult.addAll(tmp); + } + } + } else { + //make sure each cid is assigned + allocateResult = allocate(consumerGroup, clientId, mqAll, cidAll); + } + } + + } else { + allocateResult = allocateMessageQueueStrategy.allocate(consumerGroup, clientId, mqAll, cidAll); + } + } catch (Throwable e) { + log.error("QueryLoad: no assignment for group[{}] topic[{}], allocate message queue exception. strategy name: {}, ex: {}", consumerGroup, topic, strategyName, e); + return null; + } + + assignedQueueSet = new HashSet(); + if (allocateResult != null) { + assignedQueueSet.addAll(allocateResult); + } + break; + } + default: + break; + } + return assignedQueueSet; + } + + private List allocate(String consumerGroup, String currentCID, List mqAll, + List cidAll) { + if (currentCID == null || currentCID.length() < 1) { + throw new IllegalArgumentException("currentCID is empty"); + } + if (mqAll == null || mqAll.isEmpty()) { + throw new IllegalArgumentException("mqAll is null or mqAll empty"); + } + if (cidAll == null || cidAll.isEmpty()) { + throw new IllegalArgumentException("cidAll is null or cidAll empty"); + } + + List result = new ArrayList(); + if (!cidAll.contains(currentCID)) { + log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}", + consumerGroup, + currentCID, + cidAll); + return result; + } + + int index = cidAll.indexOf(currentCID); + result.add(mqAll.get(index % mqAll.size())); + return result; + } + + private RemotingCommand setMessageRequestMode(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final SetMessageRequestModeRequestBody requestBody = SetMessageRequestModeRequestBody.decode(request.getBody(), SetMessageRequestModeRequestBody.class); + + final String topic = requestBody.getTopic(); + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("retry topic is not allowed to set mode"); + return response; + } + + final String consumerGroup = requestBody.getConsumerGroup(); + + this.messageRequestModeManager.setMessageRequestMode(topic, consumerGroup, requestBody); + this.messageRequestModeManager.persist(); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/util/MsgUtil.java b/broker/src/main/java/org/apache/rocketmq/broker/util/MsgUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..80254c96adb26d5e43ed8f7aee4b342e08b920cf --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/util/MsgUtil.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.util; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; + +public final class MsgUtil { + private MsgUtil() { + } + + public static void setMessageDeliverTime(BrokerController brokerController, Message msgInner, long timeMillis) { + msgInner.setDelayTimeLevel(brokerController.getMessageStore().getScheduleMessageService().computeDelayLevel(timeMillis)); + } + + public static long getMessageDeliverTime(BrokerController brokerController, MessageExt msgInner) { + return brokerController.getMessageStore().getScheduleMessageService().computeDeliverTimestamp(msgInner.getDelayTimeLevel(), msgInner.getStoreTimestamp()); + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..c269523007adcfa9efe95e239683686c11701f5c --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import java.lang.reflect.Field; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.net.Broker2Client; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.schedule.ScheduleMessageService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class AckMessageProcessorTest { + private AckMessageProcessor ackMessageProcessor; + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + @Mock + private ChannelHandlerContext handlerContext; + @Mock + private DefaultMessageStore messageStore; + @Mock + private Channel channel; + + private String topic = "FooBar"; + private String group = "FooBarGroup"; + private ClientChannelInfo clientInfo; + @Mock + private Broker2Client broker2Client; + + @Before + public void init() throws IllegalAccessException, NoSuchFieldException { + clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0); + brokerController.setMessageStore(messageStore); + Field field = BrokerController.class.getDeclaredField("broker2Client"); + field.setAccessible(true); + field.set(brokerController, broker2Client); + ScheduleMessageService scheduleMessageService = new ScheduleMessageService(messageStore); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMessageDelayLevel("5s 10s"); + when(messageStore.getMessageStoreConfig()).thenReturn(messageStoreConfig); + scheduleMessageService.parseDelayLevel(); + when(messageStore.getScheduleMessageService()).thenReturn(scheduleMessageService); + Channel mockChannel = mock(Channel.class); + when(handlerContext.channel()).thenReturn(mockChannel); + brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig()); + ConsumerData consumerData = createConsumerData(group, topic); + brokerController.getConsumerManager().registerConsumer( + consumerData.getGroupName(), + clientInfo, + consumerData.getConsumeType(), + consumerData.getMessageModel(), + consumerData.getConsumeFromWhere(), + consumerData.getSubscriptionDataSet(), + false); + ackMessageProcessor = new AckMessageProcessor(brokerController); + } + + @Test + public void testProcessRequest_Success() throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException { + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + + int queueId = 0; + long queueOffset = 0; + long popTime = System.currentTimeMillis() - 1_000; + long invisibleTime = 30_000; + int reviveQid = 0; + String brokerName = "test_broker"; + String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid, + topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset; + AckMessageRequestHeader requestHeader = new AckMessageRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setQueueId(0); + requestHeader.setOffset(0L); + requestHeader.setConsumerGroup(group); + requestHeader.setExtraInfo(extraInfo); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand responseToReturn = ackMessageProcessor.processRequest(handlerContext, request); + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque()); + } +} \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f963c231ffb567bc4b89552c1484509b9e7a3a86 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import java.lang.reflect.Field; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.net.Broker2Client; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.schedule.ScheduleMessageService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ChangeInvisibleTimeProcessorTest { + private ChangeInvisibleTimeProcessor changeInvisibleTimeProcessor; + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + @Mock + private ChannelHandlerContext handlerContext; + @Mock + private DefaultMessageStore messageStore; + @Mock + private Channel channel; + + private String topic = "FooBar"; + private String group = "FooBarGroup"; + private ClientChannelInfo clientInfo; + @Mock + private Broker2Client broker2Client; + + @Before + public void init() throws IllegalAccessException, NoSuchFieldException { + brokerController.setMessageStore(messageStore); + Field field = BrokerController.class.getDeclaredField("broker2Client"); + field.setAccessible(true); + field.set(brokerController, broker2Client); + ScheduleMessageService scheduleMessageService = new ScheduleMessageService(messageStore); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMessageDelayLevel("5s 10s"); + when(messageStore.getMessageStoreConfig()).thenReturn(messageStoreConfig); + scheduleMessageService.parseDelayLevel(); + when(messageStore.getScheduleMessageService()).thenReturn(scheduleMessageService); + Channel mockChannel = mock(Channel.class); + when(handlerContext.channel()).thenReturn(mockChannel); + brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig()); + ConsumerData consumerData = createConsumerData(group, topic); + clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0); + brokerController.getConsumerManager().registerConsumer( + consumerData.getGroupName(), + clientInfo, + consumerData.getConsumeType(), + consumerData.getMessageModel(), + consumerData.getConsumeFromWhere(), + consumerData.getSubscriptionDataSet(), + false); + changeInvisibleTimeProcessor = new ChangeInvisibleTimeProcessor(brokerController); + } + + @Test + public void testProcessRequest_Success() throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException { + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + int queueId = 0; + long queueOffset = 0; + long popTime = System.currentTimeMillis() - 1_000; + long invisibleTime = 30_000; + int reviveQid = 0; + String brokerName = "test_broker"; + String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid, + topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset; + + ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queueId); + requestHeader.setOffset(queueOffset); + requestHeader.setConsumerGroup(group); + requestHeader.setExtraInfo(extraInfo); + requestHeader.setInvisibleTime(invisibleTime); + + final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand responseToReturn = changeInvisibleTimeProcessor.processRequest(handlerContext, request); + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque()); + } +} \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopBufferMergeServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopBufferMergeServiceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..37e79ef1a37edf09d4d39dc1418a49db15757bb7 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopBufferMergeServiceTest.java @@ -0,0 +1,108 @@ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.pop.AckMsg; +import org.apache.rocketmq.store.pop.PopCheckPoint; +import org.apache.rocketmq.store.schedule.ScheduleMessageService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class PopBufferMergeServiceTest { + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + @Mock + private PopMessageProcessor popMessageProcessor; + @Mock + private ChannelHandlerContext handlerContext; + @Mock + private DefaultMessageStore messageStore; + private ScheduleMessageService scheduleMessageService; + private ClientChannelInfo clientChannelInfo; + private String group = "FooBarGroup"; + private String topic = "FooBar"; + + @Before + public void init() throws Exception { + FieldUtils.writeField(brokerController.getBrokerConfig(), "enablePopBufferMerge", true, true); + brokerController.setMessageStore(messageStore); + popMessageProcessor = new PopMessageProcessor(brokerController); + scheduleMessageService = new ScheduleMessageService(messageStore); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMessageDelayLevel("5s 10s"); + when(messageStore.getMessageStoreConfig()).thenReturn(messageStoreConfig); + scheduleMessageService.parseDelayLevel(); + Channel mockChannel = mock(Channel.class); + brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig()); + clientChannelInfo = new ClientChannelInfo(mockChannel); + ConsumerData consumerData = createConsumerData(group, topic); + brokerController.getConsumerManager().registerConsumer( + consumerData.getGroupName(), + clientChannelInfo, + consumerData.getConsumeType(), + consumerData.getMessageModel(), + consumerData.getConsumeFromWhere(), + consumerData.getSubscriptionDataSet(), + false); + } + + @Test(timeout = 10_000) + public void testBasic() throws Exception { + PopBufferMergeService popBufferMergeService = new PopBufferMergeService(brokerController, popMessageProcessor); + popBufferMergeService.start(); + PopCheckPoint ck = new PopCheckPoint(); + ck.setBitMap(0); + int msgCnt = 1; + ck.setNum((byte) msgCnt); + long popTime = System.currentTimeMillis() - 1000; + ck.setPopTime(popTime); + int invisibleTime = 30_000; + ck.setInvisibleTime(invisibleTime); + int offset = 100; + ck.setStartOffset(offset); + ck.setCId(group); + ck.setTopic(topic); + int queueId = 0; + ck.setQueueId((byte) queueId); + + int reviveQid = 0; + long nextBeginOffset = 101L; + long ackOffset = offset; + AckMsg ackMsg = new AckMsg(); + ackMsg.setAckOffset(ackOffset); + ackMsg.setStartOffset(offset); + ackMsg.setConsumerGroup(group); + ackMsg.setTopic(topic); + ackMsg.setQueueId(queueId); + ackMsg.setPopTime(popTime); + try { + assertThat(popBufferMergeService.addCk(ck, reviveQid, ackOffset, nextBeginOffset)).isTrue(); + assertThat(popBufferMergeService.getLatestOffset(topic, group, queueId)).isEqualTo(nextBeginOffset); + Thread.sleep(1000); // wait background threads of PopBufferMergeService run for some time + assertThat(popBufferMergeService.addAk(reviveQid, ackMsg)).isTrue(); + assertThat(popBufferMergeService.getLatestOffset(topic, group, queueId)).isEqualTo(nextBeginOffset); + } finally { + popBufferMergeService.shutdown(true); + } + } +} \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..87d96b2fb99b8188bc70b9ed298c165627a9bf65 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.ConsumeInitMode; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.GetMessageStatus; +import org.apache.rocketmq.store.MappedFile; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.schedule.ScheduleMessageService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class PopMessageProcessorTest { + private PopMessageProcessor popMessageProcessor; + + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + @Mock + private ChannelHandlerContext handlerContext; + @Mock + private DefaultMessageStore messageStore; + private ScheduleMessageService scheduleMessageService; + private ClientChannelInfo clientChannelInfo; + private String group = "FooBarGroup"; + private String topic = "FooBar"; + + @Before + public void init() { + brokerController.setMessageStore(messageStore); + popMessageProcessor = new PopMessageProcessor(brokerController); + scheduleMessageService = new ScheduleMessageService(messageStore); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMessageDelayLevel("5s 10s"); + when(messageStore.getMessageStoreConfig()).thenReturn(messageStoreConfig); + scheduleMessageService.parseDelayLevel(); + when(messageStore.getScheduleMessageService()).thenReturn(scheduleMessageService); + when(messageStore.putMessage(any())).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + Channel mockChannel = mock(Channel.class); + when(mockChannel.remoteAddress()).thenReturn(new InetSocketAddress(1024)); + when(handlerContext.channel()).thenReturn(mockChannel); + brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig()); + clientChannelInfo = new ClientChannelInfo(mockChannel); + ConsumerData consumerData = createConsumerData(group, topic); + brokerController.getConsumerManager().registerConsumer( + consumerData.getGroupName(), + clientChannelInfo, + consumerData.getConsumeType(), + consumerData.getMessageModel(), + consumerData.getConsumeFromWhere(), + consumerData.getSubscriptionDataSet(), + false); + } + + @Test + public void testProcessRequest_TopicNotExist() throws RemotingCommandException { + brokerController.getTopicConfigManager().getTopicConfigTable().remove(topic); + final RemotingCommand request = createPopMsgCommand(); + RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST); + assertThat(response.getRemark()).contains("topic[" + topic + "] not exist"); + } + + @Test + public void testProcessRequest_SubNotExist() throws RemotingCommandException { + brokerController.getConsumerManager().unregisterConsumer(group, clientChannelInfo, false); + final RemotingCommand request = createPopMsgCommand(); + RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_NOT_EXIST); + assertThat(response.getRemark()).contains("consumer's group info not exist"); + } + + @Test + public void testProcessRequest_Found() throws RemotingCommandException { + GetMessageResult getMessageResult = createGetMessageResult(1); + when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(getMessageResult); + + final RemotingCommand request = createPopMsgCommand(); + RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testProcessRequest_MsgWasRemoving() throws RemotingCommandException { + GetMessageResult getMessageResult = createGetMessageResult(1); + getMessageResult.setStatus(GetMessageStatus.MESSAGE_WAS_REMOVING); + when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(getMessageResult); + + final RemotingCommand request = createPopMsgCommand(); + RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testProcessRequest_NoMsgInQueue() throws RemotingCommandException { + GetMessageResult getMessageResult = createGetMessageResult(0); + getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE); + when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(getMessageResult); + + final RemotingCommand request = createPopMsgCommand(); + RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request); + assertThat(response).isNull(); + } + + + private RemotingCommand createPopMsgCommand() { + PopMessageRequestHeader requestHeader = new PopMessageRequestHeader(); + requestHeader.setConsumerGroup(group); + requestHeader.setMaxMsgNums(30); + requestHeader.setQueueId(-1); + requestHeader.setTopic(topic); + requestHeader.setInvisibleTime(10_000); + requestHeader.setInitMode(ConsumeInitMode.MAX); + requestHeader.setOrder(false); + requestHeader.setPollTime(15_000); + requestHeader.setBornTime(System.currentTimeMillis()); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader); + request.makeCustomHeaderToNet(); + return request; + } + + + private GetMessageResult createGetMessageResult(int msgCnt) { + GetMessageResult getMessageResult = new GetMessageResult(); + getMessageResult.setStatus(GetMessageStatus.FOUND); + getMessageResult.setMinOffset(100); + getMessageResult.setMaxOffset(1024); + getMessageResult.setNextBeginOffset(516); + for (int i = 0; i < msgCnt; i++) { + ByteBuffer bb = ByteBuffer.allocate(64); + bb.putLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION, System.currentTimeMillis()); + getMessageResult.addMessage(new SelectMappedBufferResult(200, bb, 64, new MappedFile())); + } + return getMessageResult; + } +} \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..681fcc30c53a98f6b3392235bde4a46875c90bcb --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import com.google.common.collect.ImmutableSet; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.loadbalance.AssignmentManager; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.message.MessageRequestMode; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.QueryAssignmentRequestBody; +import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody; +import org.apache.rocketmq.common.protocol.body.SetMessageRequestModeRequestBody; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class QueryAssignmentProcessorTest { + private QueryAssignmentProcessor queryAssignmentProcessor; + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + + @Mock + private AssignmentManager assignmentManager; + @Mock + private ChannelHandlerContext handlerContext; + @Mock + private MessageStore messageStore; + @Mock + private Channel channel; + + private String topic = "FooBar"; + private String group = "FooBarGroup"; + private String clientId = "127.0.0.1"; + private ClientChannelInfo clientInfo; + + @Before + public void init() throws IllegalAccessException, NoSuchFieldException { + clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0); + brokerController.setMessageStore(messageStore); + doReturn(assignmentManager).when(brokerController).getAssignmentManager(); + when(assignmentManager.getTopicSubscribeInfo(topic)).thenReturn(ImmutableSet.of(new MessageQueue(topic, "broker-1", 0), new MessageQueue(topic, "broker-2", 1))); + queryAssignmentProcessor = new QueryAssignmentProcessor(brokerController); + brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig()); + ConsumerData consumerData = createConsumerData(group, topic); + brokerController.getConsumerManager().registerConsumer( + consumerData.getGroupName(), + clientInfo, + consumerData.getConsumeType(), + consumerData.getMessageModel(), + consumerData.getConsumeFromWhere(), + consumerData.getSubscriptionDataSet(), + false); + } + + @Test + public void testQueryAssignment() throws Exception { + brokerController.getProducerManager().registerProducer(group, clientInfo); + final RemotingCommand request = createQueryAssignmentRequest(); + RemotingCommand responseToReturn = queryAssignmentProcessor.processRequest(handlerContext, request); + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(responseToReturn.getBody()).isNotNull(); + QueryAssignmentResponseBody responseBody = QueryAssignmentResponseBody.decode(responseToReturn.getBody(), QueryAssignmentResponseBody.class); + assertThat(responseBody.getMessageQueueAssignments()).size().isEqualTo(2); + } + + @Test + public void testSetMessageRequestMode_Success() throws Exception { + brokerController.getProducerManager().registerProducer(group, clientInfo); + final RemotingCommand request = createSetMessageRequestModeRequest(topic); + RemotingCommand responseToReturn = queryAssignmentProcessor.processRequest(handlerContext, request); + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testSetMessageRequestMode_RetryTopic() throws Exception { + brokerController.getProducerManager().registerProducer(group, clientInfo); + final RemotingCommand request = createSetMessageRequestModeRequest(MixAll.RETRY_GROUP_TOPIC_PREFIX + topic); + RemotingCommand responseToReturn = queryAssignmentProcessor.processRequest(handlerContext, request); + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION); + } + + private RemotingCommand createQueryAssignmentRequest() { + QueryAssignmentRequestBody requestBody = new QueryAssignmentRequestBody(); + requestBody.setTopic(topic); + requestBody.setConsumerGroup(group); + requestBody.setClientId(clientId); + requestBody.setMessageModel(MessageModel.CLUSTERING); + requestBody.setStrategyName("AVG"); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_ASSIGNMENT, null); + request.setBody(requestBody.encode()); + return request; + } + + private RemotingCommand createSetMessageRequestModeRequest(String topic) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SET_MESSAGE_REQUEST_MODE, null); + + SetMessageRequestModeRequestBody requestBody = new SetMessageRequestModeRequestBody(); + requestBody.setTopic(topic); + requestBody.setConsumerGroup(group); + requestBody.setMode(MessageRequestMode.POP); + requestBody.setPopShareQueueNum(0); + request.setBody(requestBody.encode()); + + return request; + } + + private RemotingCommand createResponse(int code, RemotingCommand request) { + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(code); + response.setOpaque(request.getOpaque()); + return response; + } +} \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java index 155e692ad0bef0d7e351b6cc1a73f707067bcf0b..4c31041e7caa4b91b14eb33635b373229efba2fa 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java @@ -20,14 +20,22 @@ import java.util.ArrayList; import java.util.List; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.logging.InternalLogger; /** * Average Hashing queue algorithm */ public class AllocateMessageQueueAveragely implements AllocateMessageQueueStrategy { - private final InternalLogger log = ClientLogger.getLog(); + private InternalLogger log; + + public AllocateMessageQueueAveragely() { + log = ClientLogger.getLog(); + } + + public AllocateMessageQueueAveragely(InternalLogger log) { + this.log = log; + } @Override public List allocate(String consumerGroup, String currentCID, List mqAll, diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java index fe78f0a6bbf8f3e698c1c3a52ac2b6307b6a8ace..bd03e1ff584344c4e89381e9bc9359b72363e546 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java @@ -20,14 +20,22 @@ import java.util.ArrayList; import java.util.List; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.logging.InternalLogger; /** * Cycle average Hashing queue algorithm */ public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy { - private final InternalLogger log = ClientLogger.getLog(); + private InternalLogger log; + + public AllocateMessageQueueAveragelyByCircle() { + log = ClientLogger.getLog(); + } + + public AllocateMessageQueueAveragelyByCircle(InternalLogger log) { + this.log = log; + } @Override public List allocate(String consumerGroup, String currentCID, List mqAll, diff --git a/distribution/conf/logback_broker.xml b/distribution/conf/logback_broker.xml index 9d1a6b17618da6644c7b16a55c03e86bc2e3d109..2e4f9c668bf76299de732c304d5619ca73ac4b51 100644 --- a/distribution/conf/logback_broker.xml +++ b/distribution/conf/logback_broker.xml @@ -257,6 +257,30 @@ + + ${user.home}/logs/rocketmqlogs/pop.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/pop.%i.log + + 1 + 20 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + true @@ -330,6 +354,11 @@ + + + + + diff --git a/pom.xml b/pom.xml index 05878e5c83cae7629f48dce665f4c57a2ffb054f..065855c5b2a45629e3c0bef1b424d6faad2f64cd 100644 --- a/pom.xml +++ b/pom.xml @@ -437,7 +437,7 @@ org.mockito mockito-core - 2.23.0 + 2.28.2 test @@ -571,6 +571,11 @@ guava 19.0 + + com.googlecode.concurrentlinkedhashmap + concurrentlinkedhashmap-lru + 1.4.2 + io.openmessaging openmessaging-api diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java index f244bf4c853551ad8b4810f95a78b039ae6d239b..34f8b76a097518dc679940db72041e7186fff213 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java @@ -172,11 +172,18 @@ public class RemotingHelper { public static String parseSocketAddressAddr(SocketAddress socketAddress) { if (socketAddress != null) { + // Default toString of InetSocketAddress is "hostName/IP:port" final String addr = socketAddress.toString(); - if (addr.length() > 0) { + if (addr.contains("/")) { + String[] segments = addr.split("/"); + if (segments.length > 1) { + return segments[1]; + } + } return addr.substring(1); } + return addr; } return ""; }