Merge branch 'master' of github.com:apache/rocketmq

.travis.yml

@@ -43,6 +43,7 @@ before_script:
   - ulimit -c unlimited
 
 script:
+  - mvn verify -DskipTests
   - travis_retry mvn -B clean apache-rat:check
   - travis_retry mvn -B package jacoco:report coveralls:report
 

README.md

@@ -15,7 +15,7 @@ It offers a variety of features:
 * Messaging patterns including publish/subscribe, request/reply and streaming
 * Financial grade transactional message
 * Built-in fault tolerance and high availability configuration options base on [DLedger](https://github.com/openmessaging/openmessaging-storage-dledger)
-* A variety of cross language clients, such as Java, C/C++, Python, Go
+* A variety of cross language clients, such as Java, [C/C++](https://github.com/apache/rocketmq-client-cpp), [Python](https://github.com/apache/rocketmq-client-python), [Go](https://github.com/apache/rocketmq-client-go), [Node.js](https://github.com/apache/rocketmq-client-nodejs)
 * Pluggable transport protocols, such as TCP, SSL, AIO
 * Built-in message tracing capability, also support opentracing
 * Versatile big-data and streaming ecosytem integration

acl/pom.xml

@@ -13,7 +13,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
     <artifactId>rocketmq-acl</artifactId>
     <name>rocketmq-acl ${project.version}</name>

acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java

@@ -23,7 +23,6 @@ import java.io.FileNotFoundException;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
-import java.util.ArrayList;
 import java.util.Map;
 import java.util.SortedMap;
 import org.apache.commons.lang3.StringUtils;
@@ -206,64 +205,35 @@ public class AclUtils {
     }
 
     public static String expandIP(String netaddress, int part) {
-        boolean compress = false;
-        int compressIndex = -1;
-        String[] strArray = StringUtils.split(netaddress, ":");
-        ArrayList<Integer> indexes = new ArrayList<>();
-        for (int i = 0; i < netaddress.length(); i++) {
-            if (netaddress.charAt(i) == ':') {
-                if (indexes.size() > 0 && i - indexes.get(indexes.size() - 1) == 1) {
-                    compressIndex = i;
-                    compress = true;
-                }
-                indexes.add(i);
+        netaddress = netaddress.toUpperCase();
+        // expand netaddress
+        int separatorCount = StringUtils.countMatches(netaddress, ":");
+        int padCount = part - separatorCount;
+        if (padCount > 0) {
+            StringBuilder padStr = new StringBuilder(":");
+            for (int i = 0; i < padCount; i++) {
+                padStr.append(":");
             }
+            netaddress = StringUtils.replace(netaddress, "::", padStr.toString());
         }
 
+        // pad netaddress
+        String[] strArray = StringUtils.splitPreserveAllTokens(netaddress, ":");
         for (int i = 0; i < strArray.length; i++) {
             if (strArray[i].length() < 4) {
-                strArray[i] = "0000".substring(0, 4 - strArray[i].length()) + strArray[i];
+                strArray[i] = StringUtils.leftPad(strArray[i], 4, '0');
             }
         }
 
+        // output
         StringBuilder sb = new StringBuilder();
-        if (compress) {
-            int pos = indexes.indexOf(compressIndex);
-            int index = 0;
-            if (!netaddress.startsWith(":")) {
-                for (int i = 0; i < pos; i++) {
-                    sb.append(strArray[index]).append(":");
-                    index += 1;
-                }
-            }
-            int zeroNum = part - strArray.length;
-            if (netaddress.endsWith(":")) {
-                for (int i = 0; i < zeroNum; i++) {
-                    sb.append("0000");
-                    if (i != zeroNum - 1) {
-                        sb.append(":");
-                    }
-                }
-            } else {
-                for (int i = 0; i < zeroNum; i++) {
-                    sb.append("0000").append(":");
-                }
-                for (int i = index; i < strArray.length; i++) {
-                    sb.append(strArray[i]);
-                    if (i != strArray.length - 1) {
-                        sb.append(":");
-                    }
-                }
-            }
-        } else {
-            for (int i = 0; i < strArray.length; i++) {
-                sb.append(strArray[i]);
-                if (i != strArray.length - 1) {
-                    sb.append(":");
-                }
+        for (int i = 0; i < strArray.length; i++) {
+            sb.append(strArray[i]);
+            if (i != strArray.length - 1) {
+                sb.append(":");
             }
         }
-        return sb.toString().toUpperCase();
+        return sb.toString();
     }
 
     public static <T> T getYamlDataObject(String path, Class<T> clazz) {
@@ -308,7 +278,7 @@ public class AclUtils {
         JSONObject yamlDataObject = null;
         try {
             yamlDataObject = AclUtils.getYamlDataObject(fileName,
-                JSONObject.class);
+                    JSONObject.class);
         } catch (Exception e) {
             log.error("Convert yaml file to data object error, ", e);
             return null;

acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java

@@ -18,13 +18,6 @@ package org.apache.rocketmq.acl.plain;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.acl.common.AclConstants;
 import org.apache.rocketmq.acl.common.AclException;
@@ -39,6 +32,14 @@ import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.srvutil.FileWatchService;
 
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
 public class PlainPermissionManager {
 
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
@@ -194,9 +195,9 @@ public class PlainPermissionManager {
                     "The secretKey=%s value length should longer than 6",
                     plainAccessConfig.getSecretKey()));
             }
-            newAccountsMap.put(AclConstants.CONFIG_SECRET_KEY, (String) plainAccessConfig.getSecretKey());
+            newAccountsMap.put(AclConstants.CONFIG_SECRET_KEY, plainAccessConfig.getSecretKey());
         }
-        if (!StringUtils.isEmpty(plainAccessConfig.getWhiteRemoteAddress())) {
+        if (plainAccessConfig.getWhiteRemoteAddress() != null) {
             newAccountsMap.put(AclConstants.CONFIG_WHITE_ADDR, plainAccessConfig.getWhiteRemoteAddress());
         }
         if (!StringUtils.isEmpty(String.valueOf(plainAccessConfig.isAdmin()))) {
@@ -208,10 +209,10 @@ public class PlainPermissionManager {
         if (!StringUtils.isEmpty(plainAccessConfig.getDefaultGroupPerm())) {
             newAccountsMap.put(AclConstants.CONFIG_DEFAULT_GROUP_PERM, plainAccessConfig.getDefaultGroupPerm());
         }
-        if (plainAccessConfig.getTopicPerms() != null && !plainAccessConfig.getTopicPerms().isEmpty()) {
+        if (plainAccessConfig.getTopicPerms() != null) {
             newAccountsMap.put(AclConstants.CONFIG_TOPIC_PERMS, plainAccessConfig.getTopicPerms());
         }
-        if (plainAccessConfig.getGroupPerms() != null && !plainAccessConfig.getGroupPerms().isEmpty()) {
+        if (plainAccessConfig.getGroupPerms() != null) {
             newAccountsMap.put(AclConstants.CONFIG_GROUP_PERMS, plainAccessConfig.getGroupPerms());
         }
 

acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java

@@ -167,7 +167,7 @@ public class RemoteAddressStrategyFactory {
                 String[] strArray = StringUtils.split(remoteAddr, ".");
                 if (analysis(strArray, 1) || analysis(strArray, 2) || analysis(strArray, 3)) {
                     AclUtils.verify(remoteAddr, index - 1);
-                    StringBuffer sb = new StringBuffer();
+                    StringBuilder sb = new StringBuilder();
                     for (int j = 0; j < index; j++) {
                         sb.append(strArray[j].trim()).append(".");
                     }

acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java

@@ -202,6 +202,7 @@ public class AclUtilsTest {
 
     @Test
     public void expandIPTest() {
+        Assert.assertEquals(AclUtils.expandIP("::", 8), "0000:0000:0000:0000:0000:0000:0000:0000");
         Assert.assertEquals(AclUtils.expandIP("::1", 8), "0000:0000:0000:0000:0000:0000:0000:0001");
         Assert.assertEquals(AclUtils.expandIP("3::", 8), "0003:0000:0000:0000:0000:0000:0000:0000");
         Assert.assertEquals(AclUtils.expandIP("2::2", 8), "0002:0000:0000:0000:0000:0000:0000:0002");

acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java

@@ -19,6 +19,7 @@ package org.apache.rocketmq.acl.plain;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -524,7 +525,7 @@ public class PlainAccessValidatorTest {
         // Verify the dateversion element is correct or not
         List<Map<String, Object>> dataVersions = (List<Map<String, Object>>) readableMap.get(AclConstants.CONFIG_DATA_VERSION);
         Assert.assertEquals(1,dataVersions.get(0).get(AclConstants.CONFIG_COUNTER));
-        
+
         // Restore the backup file and flush to yaml file
         AclUtils.writeDataObject(targetFileName, backUpAclConfigMap);
     }
@@ -616,4 +617,44 @@ public class PlainAccessValidatorTest {
         Assert.assertEquals(aclConfig.getPlainAccessConfigs().size(), 2);
     }
 
+
+    @Test
+    public void updateAccessConfigEmptyPermListTest(){
+        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+        String accessKey = "updateAccessConfigEmptyPerm";
+        plainAccessConfig.setAccessKey(accessKey);
+        plainAccessConfig.setSecretKey("123456789111");
+        plainAccessConfig.setTopicPerms(Collections.singletonList("topicB=PUB"));
+        plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+        plainAccessConfig.setTopicPerms(new ArrayList<>());
+        plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+        PlainAccessConfig result = plainAccessValidator.getAllAclConfig().getPlainAccessConfigs()
+                .stream().filter(c->c.getAccessKey().equals(accessKey)).findFirst().orElse(null);
+        Assert.assertEquals(0, result.getTopicPerms().size());
+
+        plainAccessValidator.deleteAccessConfig(accessKey);
+    }
+
+    @Test
+    public void updateAccessConfigEmptyWhiteRemoteAddressTest(){
+        PlainAccessValidator plainAccessValidator = new PlainAccessValidator();
+        PlainAccessConfig plainAccessConfig = new PlainAccessConfig();
+        String accessKey = "updateAccessConfigEmptyWhiteRemoteAddress";
+        plainAccessConfig.setAccessKey(accessKey);
+        plainAccessConfig.setSecretKey("123456789111");
+        plainAccessConfig.setWhiteRemoteAddress("127.0.0.1");
+        plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+        plainAccessConfig.setWhiteRemoteAddress("");
+        plainAccessValidator.updateAccessConfig(plainAccessConfig);
+
+        PlainAccessConfig result = plainAccessValidator.getAllAclConfig().getPlainAccessConfigs()
+                .stream().filter(c->c.getAccessKey().equals(accessKey)).findFirst().orElse(null);
+        Assert.assertEquals("", result.getWhiteRemoteAddress());
+
+        plainAccessValidator.deleteAccessConfig(accessKey);
+    }
 }

broker/pom.xml

@@ -13,7 +13,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java

@@ -22,7 +22,8 @@ import java.net.SocketAddress;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
 import org.apache.rocketmq.broker.mqtrace.SendMessageHook;
@@ -60,7 +61,6 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
 
     protected final static int DLQ_NUMS_PER_GROUP = 1;
     protected final BrokerController brokerController;
-    protected final Random random = new Random(System.currentTimeMillis());
     protected final SocketAddress storeHost;
     private List<SendMessageHook> sendMessageHookList;
 
@@ -109,7 +109,7 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
         final SendMessageRequestHeader requestHeader, final byte[] body, TopicConfig topicConfig) {
         int queueIdInt = requestHeader.getQueueId();
         if (queueIdInt < 0) {
-            queueIdInt = Math.abs(this.random.nextInt() % 99999999) % topicConfig.getWriteQueueNums();
+            queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % topicConfig.getWriteQueueNums();
         }
         int sysFlag = requestHeader.getSysFlag();
 

broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java

@@ -316,8 +316,8 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements
         accessConfig.setWhiteRemoteAddress(requestHeader.getWhiteRemoteAddress());
         accessConfig.setDefaultTopicPerm(requestHeader.getDefaultTopicPerm());
         accessConfig.setDefaultGroupPerm(requestHeader.getDefaultGroupPerm());
-        accessConfig.setTopicPerms(UtilAll.string2List(requestHeader.getTopicPerms(), ","));
-        accessConfig.setGroupPerms(UtilAll.string2List(requestHeader.getGroupPerms(), ","));
+        accessConfig.setTopicPerms(UtilAll.split(requestHeader.getTopicPerms(), ","));
+        accessConfig.setGroupPerms(UtilAll.split(requestHeader.getGroupPerms(), ","));
         accessConfig.setAdmin(requestHeader.isAdmin());
         try {
 
@@ -390,7 +390,7 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements
 
         try {
             AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class);
-            if (accessValidator.updateGlobalWhiteAddrsConfig(UtilAll.string2List(requestHeader.getGlobalWhiteAddrs(), ","))) {
+            if (accessValidator.updateGlobalWhiteAddrsConfig(UtilAll.split(requestHeader.getGlobalWhiteAddrs(), ","))) {
                 response.setCode(ResponseCode.SUCCESS);
                 response.setOpaque(request.getOpaque());
                 response.markResponseType();

broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java

@@ -45,6 +45,8 @@ import org.apache.rocketmq.store.MessageExtBrokerInner;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
+import java.util.concurrent.ThreadLocalRandom;
+
 public class ReplyMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
 
@@ -125,7 +127,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen
         TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
 
         if (queueIdInt < 0) {
-            queueIdInt = Math.abs(this.random.nextInt() % 99999999) % topicConfig.getWriteQueueNums();
+            queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % topicConfig.getWriteQueueNums();
         }
 
         MessageExtBrokerInner msgInner = new MessageExtBrokerInner();

broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java

@@ -21,6 +21,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ThreadLocalRandom;
 
 import io.netty.channel.ChannelHandlerContext;
 import org.apache.rocketmq.broker.BrokerController;
@@ -48,12 +49,14 @@ import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader;
 import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.common.sysflag.TopicSysFlag;
+import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.netty.RemotingResponseCallback;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.store.MessageExtBrokerInner;
 import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
@@ -140,7 +143,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
         }
 
         String newTopic = MixAll.getRetryTopic(requestHeader.getGroup());
-        int queueIdInt = Math.abs(this.random.nextInt() % 99999999) % subscriptionGroupConfig.getRetryQueueNums();
+        int queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % subscriptionGroupConfig.getRetryQueueNums();
         int topicSysFlag = 0;
         if (requestHeader.isUnitMode()) {
             topicSysFlag = TopicSysFlag.buildSysFlag(false, true);
@@ -184,19 +187,21 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
             }
         }
 
-        if (msgExt.getReconsumeTimes() >= maxReconsumeTimes 
+        if (msgExt.getReconsumeTimes() >= maxReconsumeTimes
             || delayLevel < 0) {
             newTopic = MixAll.getDLQTopic(requestHeader.getGroup());
-            queueIdInt = Math.abs(this.random.nextInt() % 99999999) % DLQ_NUMS_PER_GROUP;
+            queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % DLQ_NUMS_PER_GROUP;
 
             topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic,
                     DLQ_NUMS_PER_GROUP,
-                    PermName.PERM_WRITE, 0);
+                    PermName.PERM_WRITE | PermName.PERM_READ, 0);
+
             if (null == topicConfig) {
                 response.setCode(ResponseCode.SYSTEM_ERROR);
                 response.setRemark("topic[" + newTopic + "] not exist");
                 return CompletableFuture.completedFuture(response);
             }
+            msgExt.setDelayTimeLevel(0);
         } else {
             if (0 == delayLevel) {
                 delayLevel = 3 + msgExt.getReconsumeTimes();
@@ -233,6 +238,12 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
                         if (correctTopic != null) {
                             backTopic = correctTopic;
                         }
+                        if (TopicValidator.RMQ_SYS_SCHEDULE_TOPIC.equals(msgInner.getTopic())) {
+                            this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic());
+                            this.brokerController.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), r.getAppendMessageResult().getWroteBytes());
+                            this.brokerController.getBrokerStatsManager().incQueuePutNums(msgInner.getTopic(), msgInner.getQueueId());
+                            this.brokerController.getBrokerStatsManager().incQueuePutSize(msgInner.getTopic(), msgInner.getQueueId(), r.getAppendMessageResult().getWroteBytes());
+                        }
                         this.brokerController.getBrokerStatsManager().incSendBackNums(requestHeader.getGroup(), backTopic);
                         response.setCode(ResponseCode.SUCCESS);
                         response.setRemark(null);
@@ -351,13 +362,14 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
             int reconsumeTimes = requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes();
             if (reconsumeTimes >= maxReconsumeTimes) {
                 newTopic = MixAll.getDLQTopic(groupName);
-                int queueIdInt = Math.abs(this.random.nextInt() % 99999999) % DLQ_NUMS_PER_GROUP;
+                int queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % DLQ_NUMS_PER_GROUP;
                 topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic,
                     DLQ_NUMS_PER_GROUP,
-                    PermName.PERM_WRITE, 0
+                    PermName.PERM_WRITE | PermName.PERM_READ, 0
                 );
                 msg.setTopic(newTopic);
                 msg.setQueueId(queueIdInt);
+                msg.setDelayTimeLevel(0);
                 if (null == topicConfig) {
                     response.setCode(ResponseCode.SYSTEM_ERROR);
                     response.setRemark("topic[" + newTopic + "] not exist");
@@ -407,7 +419,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
         TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
 
         if (queueIdInt < 0) {
-            queueIdInt = Math.abs(this.random.nextInt() % 99999999) % topicConfig.getWriteQueueNums();
+            queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % topicConfig.getWriteQueueNums();
         }
 
         MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
@@ -513,6 +525,11 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
         String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);
         if (sendOK) {
 
+            if (TopicValidator.RMQ_SYS_SCHEDULE_TOPIC.equals(msg.getTopic())) {
+                this.brokerController.getBrokerStatsManager().incQueuePutNums(msg.getTopic(), msg.getQueueId(), putMessageResult.getAppendMessageResult().getMsgNum(), 1);
+                this.brokerController.getBrokerStatsManager().incQueuePutSize(msg.getTopic(), msg.getQueueId(), putMessageResult.getAppendMessageResult().getWroteBytes());
+            }
+
             this.brokerController.getBrokerStatsManager().incTopicPutNums(msg.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1);
             this.brokerController.getBrokerStatsManager().incTopicPutSize(msg.getTopic(),
                 putMessageResult.getAppendMessageResult().getWroteBytes());
@@ -626,8 +643,12 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
     }
 
     private String diskUtil() {
-        String storePathPhysic = this.brokerController.getMessageStoreConfig().getStorePathCommitLog();
-        double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic);
+        double physicRatio = 100;
+        String storePath = this.brokerController.getMessageStoreConfig().getStorePathCommitLog();
+        String[] paths = storePath.trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+        for (String storePathPhysic : paths) {
+            physicRatio = Math.min(physicRatio, UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic));
+        }
 
         String storePathLogis =
             StorePathConfigHelper.getStorePathConsumeQueue(this.brokerController.getMessageStoreConfig().getStorePathRootDir());
@@ -658,7 +679,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
     }
 
     private int randomQueueId(int writeQueueNums) {
-        return (this.random.nextInt() % 99999999) % writeQueueNums;
+        return ThreadLocalRandom.current().nextInt(99999999) % writeQueueNums;
     }
 
     private RemotingCommand preSend(ChannelHandlerContext ctx, RemotingCommand request,

broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java

@@ -17,7 +17,6 @@
 package org.apache.rocketmq.broker.transaction;
 
 import io.netty.channel.Channel;
-import java.util.Random;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.message.MessageConst;
@@ -40,7 +39,6 @@ public abstract class AbstractTransactionalMessageCheckListener {
 
     //queue nums of topic TRANS_CHECK_MAX_TIME_TOPIC
     protected final static int TCMT_QUEUE_NUMS = 1;
-    protected final Random random = new Random(System.currentTimeMillis());
 
     private static ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(2000), new ThreadFactory() {
         @Override

broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListener.java

@@ -30,6 +30,8 @@ import org.apache.rocketmq.store.MessageExtBrokerInner;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
 
+import java.util.concurrent.ThreadLocalRandom;
+
 public class DefaultTransactionalMessageCheckListener extends AbstractTransactionalMessageCheckListener {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
 
@@ -58,7 +60,7 @@ public class DefaultTransactionalMessageCheckListener extends AbstractTransactio
 
     private MessageExtBrokerInner toMessageExtBrokerInner(MessageExt msgExt) {
         TopicConfig topicConfig = this.getBrokerController().getTopicConfigManager().createTopicOfTranCheckMaxTime(TCMT_QUEUE_NUMS, PermName.PERM_READ | PermName.PERM_WRITE);
-        int queueId = Math.abs(random.nextInt() % 99999999) % TCMT_QUEUE_NUMS;
+        int queueId = ThreadLocalRandom.current().nextInt(99999999) % TCMT_QUEUE_NUMS;
         MessageExtBrokerInner inner = new MessageExtBrokerInner();
         inner.setTopic(topicConfig.getTopicName());
         inner.setBody(msgExt.getBody());

client/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -47,10 +47,6 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
-        <dependency>
-            <groupId>commons-codec</groupId>
-            <artifactId>commons-codec</artifactId>
-        </dependency>
         <dependency>
             <groupId>io.opentracing</groupId>
             <artifactId>opentracing-api</artifactId>

client/src/main/java/org/apache/rocketmq/client/ClientConfig.java

@@ -60,6 +60,8 @@ public class ClientConfig {
 
     private boolean useTLS = TlsSystemConfig.tlsEnable;
 
+    private int mqClientApiTimeout = 3 * 1000;
+
     private LanguageCode language = LanguageCode.JAVA;
 
     public String buildMQClientId() {
@@ -298,6 +300,13 @@ public class ClientConfig {
         this.accessChannel = accessChannel;
     }
 
+    public int getMqClientApiTimeout() {
+        return mqClientApiTimeout;
+    }
+
+    public void setMqClientApiTimeout(int mqClientApiTimeout) {
+        this.mqClientApiTimeout = mqClientApiTimeout;
+    }
 
     @Override
     public String toString() {
@@ -305,6 +314,6 @@ public class ClientConfig {
             + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + ", pollNameServerInterval=" + pollNameServerInterval
             + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval
             + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled="
-            + vipChannelEnabled + ", useTLS=" + useTLS + ", language=" + language.name() + ", namespace=" + namespace + "]";
+            + vipChannelEnabled + ", useTLS=" + useTLS + ", language=" + language.name() + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout + "]";
     }
 }

client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java

@@ -535,7 +535,9 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon
     private void setTraceDispatcher() {
         if (isEnableMsgTrace()) {
             try {
-                this.traceDispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, customizedTraceTopic, null);
+                AsyncTraceDispatcher traceDispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, customizedTraceTopic, null);
+                traceDispatcher.getTraceProducer().setUseTLS(this.isUseTLS());
+                this.traceDispatcher = traceDispatcher;
                 this.defaultLitePullConsumerImpl.registerConsumeMessageHook(
                     new ConsumeMessageTraceHookImpl(traceDispatcher));
             } catch (Throwable e) {

client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java

@@ -412,7 +412,15 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
     public void createTopic(String key, String newTopic, int queueNum) throws MQClientException {
         createTopic(key, withNamespace(newTopic), queueNum, 0);
     }
-
+    
+    @Override
+    public void setUseTLS(boolean useTLS) {
+        super.setUseTLS(useTLS);
+        if (traceDispatcher != null && traceDispatcher instanceof AsyncTraceDispatcher) {
+            ((AsyncTraceDispatcher) traceDispatcher).getTraceProducer().setUseTLS(useTLS);
+        }
+    }
+    
     /**
      * This method will be removed in a certain version after April 5, 2020, so please do not use this method.
      */

client/src/main/java/org/apache/rocketmq/client/consumer/MessageSelector.java

@@ -47,7 +47,7 @@ public class MessageSelector {
     }
 
     /**
-     * Use SLQ92 to select message.
+     * Use SQL92 to select message.
      *
      * @param sql if null or empty, will be treated as select all message.
      */

client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java

@@ -135,6 +135,8 @@ import org.apache.rocketmq.common.protocol.header.UpdateGlobalWhiteAddrsConfigRe
 import org.apache.rocketmq.common.protocol.header.ViewBrokerStatsDataRequestHeader;
 import org.apache.rocketmq.common.protocol.header.ViewMessageRequestHeader;
 import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterMessageFilterClassRequestHeader;
+import org.apache.rocketmq.common.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader;
+import org.apache.rocketmq.common.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader;
 import org.apache.rocketmq.common.protocol.header.namesrv.DeleteKVConfigRequestHeader;
 import org.apache.rocketmq.common.protocol.header.namesrv.GetKVConfigRequestHeader;
 import org.apache.rocketmq.common.protocol.header.namesrv.GetKVConfigResponseHeader;
@@ -305,8 +307,8 @@ public class MQClientAPIImpl {
         requestHeader.setDefaultGroupPerm(plainAccessConfig.getDefaultGroupPerm());
         requestHeader.setDefaultTopicPerm(plainAccessConfig.getDefaultTopicPerm());
         requestHeader.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress());
-        requestHeader.setTopicPerms(UtilAll.list2String(plainAccessConfig.getTopicPerms(), ","));
-        requestHeader.setGroupPerms(UtilAll.list2String(plainAccessConfig.getGroupPerms(), ","));
+        requestHeader.setTopicPerms(UtilAll.join(plainAccessConfig.getTopicPerms(), ","));
+        requestHeader.setGroupPerms(UtilAll.join(plainAccessConfig.getGroupPerms(), ","));
 
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_ACL_CONFIG, requestHeader);
 
@@ -1428,6 +1430,28 @@ public class MQClientAPIImpl {
         throw new MQClientException(response.getCode(), response.getRemark());
     }
 
+    public int addWritePermOfBroker(final String nameSrvAddr, String brokerName, final long timeoutMillis)
+            throws RemotingCommandException,
+            RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQClientException {
+        AddWritePermOfBrokerRequestHeader requestHeader = new AddWritePermOfBrokerRequestHeader();
+        requestHeader.setBrokerName(brokerName);
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ADD_WRITE_PERM_OF_BROKER, requestHeader);
+
+        RemotingCommand response = this.remotingClient.invokeSync(nameSrvAddr, request, timeoutMillis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                AddWritePermOfBrokerResponseHeader responseHeader =
+                        (AddWritePermOfBrokerResponseHeader) response.decodeCommandCustomHeader(AddWritePermOfBrokerResponseHeader.class);
+                return responseHeader.getAddTopicCount();
+            }
+            default:
+                break;
+        }
+        throw new MQClientException(response.getCode(), response.getRemark());
+    }
+
     public void deleteTopicInBroker(final String addr, final String topic, final long timeoutMillis)
         throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
         DeleteTopicRequestHeader requestHeader = new DeleteTopicRequestHeader();

client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java

@@ -593,8 +593,19 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
         }
         final Object objLock = messageQueueLock.fetchLockObject(messageQueue);
         synchronized (objLock) {
-            assignedMessageQueue.setSeekOffset(messageQueue, offset);
             clearMessageQueueInCache(messageQueue);
+
+            PullTaskImpl oldPullTaskImpl = this.taskTable.get(messageQueue);
+            if (oldPullTaskImpl != null) {
+                oldPullTaskImpl.tryInterrupt();
+                this.taskTable.remove(messageQueue);
+            }
+            assignedMessageQueue.setSeekOffset(messageQueue, offset);
+            if (!this.taskTable.containsKey(messageQueue)) {
+                PullTaskImpl pullTask = new PullTaskImpl(messageQueue);
+                this.taskTable.put(messageQueue, pullTask);
+                this.scheduledThreadPoolExecutor.schedule(pullTask, 0, TimeUnit.MILLISECONDS);
+            }
         }
     }
 
@@ -718,16 +729,29 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
     public class PullTaskImpl implements Runnable {
         private final MessageQueue messageQueue;
         private volatile boolean cancelled = false;
+        private Thread currentThread;
 
         public PullTaskImpl(final MessageQueue messageQueue) {
             this.messageQueue = messageQueue;
         }
 
+        public void tryInterrupt() {
+            setCancelled(true);
+            if (currentThread == null) {
+                return;
+            }
+            if (!currentThread.isInterrupted()) {
+                currentThread.interrupt();
+            }
+        }
+
         @Override
         public void run() {
 
             if (!this.isCancelled()) {
 
+                this.currentThread = Thread.currentThread();
+
                 if (assignedMessageQueue.isPaused(messageQueue)) {
                     scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_PAUSE, TimeUnit.MILLISECONDS);
                     log.debug("Message Queue: {} has been paused!", messageQueue);
@@ -803,7 +827,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
                     } else {
                         subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL);
                     }
-                    
+
                     PullResult pullResult = pull(messageQueue, subscriptionData, offset, defaultLitePullConsumer.getPullBatchSize());
                     if (this.isCancelled() || processQueue.isDropped()) {
                         return;

client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java

@@ -447,7 +447,7 @@ public class MQClientInstance {
                 if (addr != null) {
                     try {
                         this.getMQClientAPIImpl().checkClientInBroker(
-                            addr, entry.getKey(), this.clientId, subscriptionData, 3 * 1000
+                            addr, entry.getKey(), this.clientId, subscriptionData, clientConfig.getMqClientApiTimeout()
                         );
                     } catch (Exception e) {
                         if (e instanceof MQClientException) {
@@ -554,7 +554,7 @@ public class MQClientInstance {
                             }
 
                             try {
-                                int version = this.mQClientAPIImpl.sendHearbeat(addr, heartbeatData, 3000);
+                                int version = this.mQClientAPIImpl.sendHearbeat(addr, heartbeatData, clientConfig.getMqClientApiTimeout());
                                 if (!this.brokerVersionTable.containsKey(brokerName)) {
                                     this.brokerVersionTable.put(brokerName, new HashMap<String, Integer>(4));
                                 }
@@ -610,7 +610,7 @@ public class MQClientInstance {
                     TopicRouteData topicRouteData;
                     if (isDefault && defaultMQProducer != null) {
                         topicRouteData = this.mQClientAPIImpl.getDefaultTopicRouteInfoFromNameServer(defaultMQProducer.getCreateTopicKey(),
-                            1000 * 3);
+                            clientConfig.getMqClientApiTimeout());
                         if (topicRouteData != null) {
                             for (QueueData data : topicRouteData.getQueueDatas()) {
                                 int queueNums = Math.min(defaultMQProducer.getDefaultTopicQueueNums(), data.getReadQueueNums());
@@ -619,7 +619,7 @@ public class MQClientInstance {
                             }
                         }
                     } else {
-                        topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, 1000 * 3);
+                        topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, clientConfig.getMqClientApiTimeout());
                     }
                     if (topicRouteData != null) {
                         TopicRouteData old = this.topicRouteTable.get(topic);
@@ -894,7 +894,7 @@ public class MQClientInstance {
                     String addr = entry1.getValue();
                     if (addr != null) {
                         try {
-                            this.mQClientAPIImpl.unregisterClient(addr, this.clientId, producerGroup, consumerGroup, 3000);
+                            this.mQClientAPIImpl.unregisterClient(addr, this.clientId, producerGroup, consumerGroup, clientConfig.getMqClientApiTimeout());
                             log.info("unregister client[Producer: {} Consumer: {}] from broker[{} {} {}] success", producerGroup, consumerGroup, brokerName, entry1.getKey(), addr);
                         } catch (RemotingException e) {
                             log.error("unregister client exception from broker: " + addr, e);
@@ -1064,7 +1064,7 @@ public class MQClientInstance {
 
         if (null != brokerAddr) {
             try {
-                return this.mQClientAPIImpl.getConsumerIdListByGroup(brokerAddr, group, 3000);
+                return this.mQClientAPIImpl.getConsumerIdListByGroup(brokerAddr, group, clientConfig.getMqClientApiTimeout());
             } catch (Exception e) {
                 log.warn("getConsumerIdListByGroup exception, " + brokerAddr + " " + group, e);
             }

client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java

@@ -82,7 +82,6 @@ import org.apache.rocketmq.common.message.MessageId;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.message.MessageType;
 import org.apache.rocketmq.common.protocol.NamespaceUtil;
-import org.apache.rocketmq.common.protocol.ResponseCode;
 import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader;
 import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
 import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
@@ -643,20 +642,14 @@ public class DefaultMQProducerImpl implements MQProducerInner {
                         log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e);
                         log.warn(msg.toString());
                         exception = e;
-                        switch (e.getResponseCode()) {
-                            case ResponseCode.TOPIC_NOT_EXIST:
-                            case ResponseCode.SERVICE_NOT_AVAILABLE:
-                            case ResponseCode.SYSTEM_ERROR:
-                            case ResponseCode.NO_PERMISSION:
-                            case ResponseCode.NO_BUYER_ID:
-                            case ResponseCode.NOT_IN_CURRENT_UNIT:
-                                continue;
-                            default:
-                                if (sendResult != null) {
-                                    return sendResult;
-                                }
+                        if (this.defaultMQProducer.getRetryResponseCodes().contains(e.getResponseCode())) {
+                            continue;
+                        } else {
+                            if (sendResult != null) {
+                                return sendResult;
+                            }
 
-                                throw e;
+                            throw e;
                         }
                     } catch (InterruptedException e) {
                         endTimestamp = System.currentTimeMillis();

client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java

@@ -16,8 +16,11 @@
  */
 package org.apache.rocketmq.client.producer;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.concurrent.ExecutorService;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.QueryResult;
@@ -39,6 +42,7 @@ import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageId;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.ResponseCode;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.remoting.RPCHook;
@@ -63,6 +67,15 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
      */
     protected final transient DefaultMQProducerImpl defaultMQProducerImpl;
     private final InternalLogger log = ClientLogger.getLog();
+    private final Set<Integer> retryResponseCodes = new CopyOnWriteArraySet<Integer>(Arrays.asList(
+            ResponseCode.TOPIC_NOT_EXIST,
+            ResponseCode.SERVICE_NOT_AVAILABLE,
+            ResponseCode.SYSTEM_ERROR,
+            ResponseCode.NO_PERMISSION,
+            ResponseCode.NO_BUYER_ID,
+            ResponseCode.NOT_IN_CURRENT_UNIT
+    ));
+
     /**
      * Producer group conceptually aggregates all producer instances of exactly same role, which is particularly
      * important when transactional messages are involved. </p>
@@ -158,22 +171,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
      */
     public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace,
         final String customizedTraceTopic) {
-        this.producerGroup = producerGroup;
-        defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook);
-        //if client open the message trace feature
-        if (enableMsgTrace) {
-            try {
-                AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(producerGroup, TraceDispatcher.Type.PRODUCE, customizedTraceTopic, rpcHook);
-                dispatcher.setHostProducer(this.defaultMQProducerImpl);
-                traceDispatcher = dispatcher;
-                this.defaultMQProducerImpl.registerSendMessageHook(
-                    new SendMessageTraceHookImpl(traceDispatcher));
-                this.defaultMQProducerImpl.registerEndTransactionHook(
-                    new EndTransactionTraceHookImpl(traceDispatcher));
-            } catch (Throwable e) {
-                log.error("system mqtrace hook init failed ,maybe can't send msg trace data");
-            }
-        }
+        this(null, producerGroup, rpcHook, enableMsgTrace, customizedTraceTopic);
     }
 
     /**
@@ -251,9 +249,9 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
         if (enableMsgTrace) {
             try {
                 AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(producerGroup, TraceDispatcher.Type.PRODUCE, customizedTraceTopic, rpcHook);
-                dispatcher.setHostProducer(this.getDefaultMQProducerImpl());
+                dispatcher.setHostProducer(this.defaultMQProducerImpl);
                 traceDispatcher = dispatcher;
-                this.getDefaultMQProducerImpl().registerSendMessageHook(
+                this.defaultMQProducerImpl.registerSendMessageHook(
                     new SendMessageTraceHookImpl(traceDispatcher));
                 this.defaultMQProducerImpl.registerEndTransactionHook(
                     new EndTransactionTraceHookImpl(traceDispatcher));
@@ -263,6 +261,14 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
         }
     }
 
+    @Override
+    public void setUseTLS(boolean useTLS) {
+        super.setUseTLS(useTLS);
+        if (traceDispatcher != null && traceDispatcher instanceof AsyncTraceDispatcher) {
+            ((AsyncTraceDispatcher) traceDispatcher).getTraceProducer().setUseTLS(useTLS);
+        }
+    }
+    
     /**
      * Start this producer instance. </p>
      *
@@ -965,6 +971,15 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
         this.defaultMQProducerImpl.setAsyncSenderExecutor(asyncSenderExecutor);
     }
 
+    /**
+     * Add response code for retrying.
+     *
+     * @param responseCode response code, {@link ResponseCode}
+     */
+    public void addRetryResponseCode(int responseCode) {
+        this.retryResponseCodes.add(responseCode);
+    }
+
     private MessageBatch batch(Collection<Message> msgs) throws MQClientException {
         MessageBatch msgBatch;
         try {
@@ -1095,4 +1110,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer {
         return traceDispatcher;
     }
 
+    public Set<Integer> getRetryResponseCodes() {
+        return retryResponseCodes;
+    }
 }

client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java

@@ -188,8 +188,9 @@ public class TraceDataEncoder {
                         .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)//
                         .append(ctx.isSuccess()).append(TraceConstants.CONTENT_SPLITOR)//
                         .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)//
-                        .append(ctx.getContextCode()).append(TraceConstants.FIELD_SPLITOR);
-
+                        .append(ctx.getContextCode()).append(TraceConstants.CONTENT_SPLITOR)
+                        .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)
+                        .append(ctx.getGroupName()).append(TraceConstants.FIELD_SPLITOR);
                 }
             }
             break;

client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java

@@ -17,9 +17,9 @@
 
 package org.apache.rocketmq.client.trace;
 
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
-import org.apache.commons.codec.Charsets;
 import org.apache.rocketmq.common.message.MessageExt;
 
 public class TraceView {
@@ -40,7 +40,7 @@ public class TraceView {
 
     public static List<TraceView> decodeFromTraceTransData(String key, MessageExt messageExt) {
         List<TraceView> messageTraceViewList = new ArrayList<TraceView>();
-        String messageBody = new String(messageExt.getBody(), Charsets.UTF_8);
+        String messageBody = new String(messageExt.getBody(), StandardCharsets.UTF_8);
         if (messageBody == null || messageBody.length() <= 0) {
             return messageTraceViewList;
         }

client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java

@@ -16,7 +16,6 @@
  */
 package org.apache.rocketmq.client.impl;
 
-import java.lang.reflect.Field;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
@@ -29,9 +28,11 @@ import org.apache.rocketmq.client.producer.SendStatus;
 import org.apache.rocketmq.common.PlainAccessConfig;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.protocol.RequestCode;
 import org.apache.rocketmq.common.protocol.ResponseCode;
 import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
 import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader;
+import org.apache.rocketmq.common.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader;
 import org.apache.rocketmq.remoting.InvokeCallback;
 import org.apache.rocketmq.remoting.RemotingClient;
 import org.apache.rocketmq.remoting.exception.RemotingException;
@@ -48,6 +49,8 @@ import org.mockito.invocation.InvocationOnMock;
 import org.mockito.junit.MockitoJUnitRunner;
 import org.mockito.stubbing.Answer;
 
+import java.lang.reflect.Field;
+
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;
 import static org.mockito.ArgumentMatchers.any;
@@ -442,4 +445,27 @@ public class MQClientAPIImplTest {
         requestHeader.setMaxReconsumeTimes(10);
         return requestHeader;
     }
+
+    @Test
+    public void testAddWritePermOfBroker() throws Exception {
+        doAnswer(new Answer() {
+            @Override
+            public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
+                RemotingCommand request = invocationOnMock.getArgument(1);
+                if (request.getCode() != RequestCode.ADD_WRITE_PERM_OF_BROKER) {
+                    return null;
+                }
+
+                RemotingCommand response = RemotingCommand.createResponseCommand(AddWritePermOfBrokerResponseHeader.class);
+                AddWritePermOfBrokerResponseHeader responseHeader = (AddWritePermOfBrokerResponseHeader) response.readCustomHeader();
+                response.setCode(ResponseCode.SUCCESS);
+                responseHeader.setAddTopicCount(7);
+                response.addExtField("addTopicCount", String.valueOf(responseHeader.getAddTopicCount()));
+                return response;
+            }
+        }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong());
+
+        int topicCnt = mqClientAPI.addWritePermOfBroker("127.0.0.1", "default-broker", 1000);
+        assertThat(topicCnt).isEqualTo(7);
+    }
 }
\ No newline at end of file

client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java

@@ -178,7 +178,7 @@ public class ConsumeMessageConcurrentlyServiceTest {
         StatsItemSet itemSet = (StatsItemSet)statItmeSetField.get(mgr);
         StatsItem item = itemSet.getAndCreateStatsItem(topic + "@" + pushConsumer.getDefaultMQPushConsumerImpl().groupName());
 
-        assertThat(item.getValue().get()).isGreaterThan(0L);
+        assertThat(item.getValue().sum()).isGreaterThan(0L);
         MessageExt msg = messageAtomic.get();
         assertThat(msg).isNotNull();
         assertThat(msg.getTopic()).isEqualTo(topic);

client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java

@@ -69,6 +69,7 @@ import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -235,6 +236,14 @@ public class DefaultMQConsumerWithTraceTest {
         assertThat(msg.getTopic()).isEqualTo(topic);
         assertThat(msg.getBody()).isEqualTo(new byte[] {'a'});
     }
+    
+    @Test
+    public void testPushConsumerWithTraceTLS() {
+        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("consumerGroup", true);
+        consumer.setUseTLS(true);
+        AsyncTraceDispatcher asyncTraceDispatcher = (AsyncTraceDispatcher) consumer.getTraceDispatcher();
+        Assert.assertTrue(asyncTraceDispatcher.getTraceProducer().isUseTLS());
+    }
 
     private PullRequest createPullRequest() {
         PullRequest pullRequest = new PullRequest();

client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java

@@ -52,6 +52,7 @@ import org.apache.rocketmq.common.protocol.route.QueueData;
 import org.apache.rocketmq.common.protocol.route.TopicRouteData;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.remoting.RPCHook;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -146,6 +147,15 @@ public class DefaultMQLitePullConsumerWithTraceTest {
         }
     }
 
+    @Test
+    public void testLitePullConsumerWithTraceTLS() throws Exception {
+        DefaultLitePullConsumer consumer = new DefaultLitePullConsumer("consumerGroup");
+        consumer.setUseTLS(true);
+        consumer.setEnableMsgTrace(true);
+        consumer.start();
+        AsyncTraceDispatcher asyncTraceDispatcher = (AsyncTraceDispatcher) consumer.getTraceDispatcher();
+        Assert.assertTrue(asyncTraceDispatcher.getTraceProducer().isUseTLS());
+    }
 
     private DefaultLitePullConsumer createLitePullConsumerWithDefaultTraceTopic() throws Exception {
         DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis());
@@ -302,4 +312,4 @@ public class DefaultMQLitePullConsumerWithTraceTest {
         doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString());
     }
 
-}
+}
\ No newline at end of file

client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java

@@ -39,6 +39,7 @@ import org.apache.rocketmq.common.protocol.route.TopicRouteData;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -143,6 +144,15 @@ public class DefaultMQProducerWithTraceTest {
 
     }
 
+    
+    @Test
+    public void testProducerWithTraceTLS() {
+        DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp, true);
+        producer.setUseTLS(true);
+        AsyncTraceDispatcher asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher();
+        Assert.assertTrue(asyncTraceDispatcher.getTraceProducer().isUseTLS());
+    }
+    
     @After
     public void terminate() {
         producer.shutdown();

client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java

@@ -189,6 +189,8 @@ public class TraceDataEncoderTest {
         subAfterContext.setRequestId("3455848576927");
         subAfterContext.setCostTime(20);
         subAfterContext.setSuccess(true);
+        subAfterContext.setTimeStamp(1625883640000L);
+        subAfterContext.setGroupName("GroupName-test");
         subAfterContext.setContextCode(98623046);
         TraceBean bean = new TraceBean();
         bean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000");
@@ -200,7 +202,7 @@ public class TraceDataEncoderTest {
         String transData = traceTransferBean.getTransData();
         Assert.assertNotNull(transData);
         String[] items = transData.split(String.valueOf(TraceConstants.CONTENT_SPLITOR));
-        Assert.assertEquals(7, items.length);
+        Assert.assertEquals(9, items.length);
 
     }
 

client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java

@@ -17,12 +17,12 @@
 
 package org.apache.rocketmq.client.trace;
 
-import org.apache.commons.codec.Charsets;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageType;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 public class TraceViewTest {
@@ -46,7 +46,7 @@ public class TraceViewTest {
             .append(true).append(TraceConstants.FIELD_SPLITOR)
             .toString();
         MessageExt message = new MessageExt();
-        message.setBody(messageBody.getBytes(Charsets.UTF_8));
+        message.setBody(messageBody.getBytes(StandardCharsets.UTF_8));
         String key = "AC1415116D1418B4AAC217FE1B4E0000";
         List<TraceView> traceViews = TraceView.decodeFromTraceTransData(key, message);
         Assert.assertEquals(traceViews.size(), 1);

common/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

common/src/main/java/org/apache/rocketmq/common/MQVersion.java

@@ -18,7 +18,7 @@ package org.apache.rocketmq.common;
 
 public class MQVersion {
 
-    public static final int CURRENT_VERSION = Version.V4_9_1.ordinal();
+    public static final int CURRENT_VERSION = Version.V4_9_2.ordinal();
 
     public static String getVersionDesc(int value) {
         int length = Version.values().length;

common/src/main/java/org/apache/rocketmq/common/MixAll.java

@@ -58,6 +58,7 @@ public class MixAll {
     public static final String DEFAULT_PRODUCER_GROUP = "DEFAULT_PRODUCER";
     public static final String DEFAULT_CONSUMER_GROUP = "DEFAULT_CONSUMER";
     public static final String TOOLS_CONSUMER_GROUP = "TOOLS_CONSUMER";
+    public static final String SCHEDULE_CONSUMER_GROUP = "SCHEDULE_CONSUMER";
     public static final String FILTERSRV_CONSUMER_GROUP = "FILTERSRV_CONSUMER";
     public static final String MONITOR_CONSUMER_GROUP = "__MONITOR_CONSUMER";
     public static final String CLIENT_INNER_PRODUCER_GROUP = "CLIENT_INNER_PRODUCER";

common/src/main/java/org/apache/rocketmq/common/UtilAll.java

@@ -21,7 +21,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
 import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
@@ -39,7 +38,6 @@ import java.util.Map;
 import java.util.zip.CRC32;
 import java.util.zip.DeflaterOutputStream;
 import java.util.zip.InflaterInputStream;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.validator.routines.InetAddressValidator;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.InternalLogger;
@@ -53,12 +51,11 @@ public class UtilAll {
     public static final String YYYY_MM_DD_HH_MM_SS_SSS = "yyyy-MM-dd#HH:mm:ss:SSS";
     public static final String YYYYMMDDHHMMSS = "yyyyMMddHHmmss";
     final static char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray();
+    final static String HOST_NAME = ManagementFactory.getRuntimeMXBean().getName(); // format: "pid@hostname"
 
     public static int getPid() {
-        RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
-        String name = runtime.getName(); // format: "pid@hostname"
         try {
-            return Integer.parseInt(name.substring(0, name.indexOf('@')));
+            return Integer.parseInt(HOST_NAME.substring(0, HOST_NAME.indexOf('@')));
         } catch (Exception e) {
             return -1;
         }
@@ -199,6 +196,11 @@ public class UtilAll {
             cal.get(Calendar.SECOND));
     }
 
+    public static boolean isPathExists(final String path) {
+        File file = new File(path);
+        return file.exists();
+    }
+
     public static double getDiskPartitionSpaceUsedPercent(final String path) {
         if (null == path || path.isEmpty()) {
             log.error("Error when measuring disk space usage, path is null or empty, path : {}", path);
@@ -218,10 +220,15 @@ public class UtilAll {
             long totalSpace = file.getTotalSpace();
 
             if (totalSpace > 0) {
-                long freeSpace = file.getFreeSpace();
-                long usedSpace = totalSpace - freeSpace;
-
-                return usedSpace / (double) totalSpace;
+                long usedSpace = totalSpace - file.getFreeSpace();
+                long usableSpace = file.getUsableSpace();
+                long entireSpace = usedSpace + usableSpace;
+                long roundNum = 0;
+                if (usedSpace * 100 % entireSpace != 0) {
+                    roundNum = 1;
+                }
+                long result = usedSpace * 100 / entireSpace + roundNum;
+                return result / 100.0;
             }
         } catch (Exception e) {
             log.error("Error when measuring disk space usage, got exception: :", e);
@@ -461,7 +468,7 @@ public class UtilAll {
         if (ip.length != 4) {
             throw new RuntimeException("illegal ipv4 bytes");
         }
-    
+
         InetAddressValidator validator = InetAddressValidator.getInstance();
         return validator.isValidInet4Address(ipToIPv4Str(ip));
     }
@@ -561,27 +568,28 @@ public class UtilAll {
         }
     }
 
-    public static String list2String(List<String> list, String splitor) {
-        if (list == null || list.size() == 0) {
+    public static String join(List<String> list, String splitter) {
+        if (list == null) {
             return null;
         }
-        StringBuffer str = new StringBuffer();
+
+        StringBuilder str = new StringBuilder();
         for (int i = 0; i < list.size(); i++) {
             str.append(list.get(i));
             if (i == list.size() - 1) {
-                continue;
+                break;
             }
-            str.append(splitor);
+            str.append(splitter);
         }
         return str.toString();
     }
 
-    public static List<String> string2List(String str, String splitor) {
-        if (StringUtils.isEmpty(str)) {
+    public static List<String> split(String str, String splitter) {
+        if (str == null) {
             return null;
         }
 
-        String[] addrArray = str.split(splitor);
+        String[] addrArray = str.split(splitter);
         return Arrays.asList(addrArray);
     }
 }

common/src/main/java/org/apache/rocketmq/common/message/Message.java

@@ -43,11 +43,13 @@ public class Message implements Serializable {
         this.flag = flag;
         this.body = body;
 
-        if (tags != null && tags.length() > 0)
+        if (tags != null && tags.length() > 0) {
             this.setTags(tags);
+        }
 
-        if (keys != null && keys.length() > 0)
+        if (keys != null && keys.length() > 0) {
             this.setKeys(keys);
+        }
 
         this.setWaitStoreMsgOK(waitStoreMsgOK);
     }
@@ -127,7 +129,7 @@ public class Message implements Serializable {
     }
 
     public void setKeys(Collection<String> keys) {
-        StringBuffer sb = new StringBuffer();
+        StringBuilder sb = new StringBuilder();
         for (String k : keys) {
             sb.append(k);
             sb.append(MessageConst.KEY_SEPARATOR);
@@ -151,8 +153,9 @@ public class Message implements Serializable {
 
     public boolean isWaitStoreMsgOK() {
         String result = this.getProperty(MessageConst.PROPERTY_WAIT_STORE_MSG_OK);
-        if (null == result)
+        if (null == result) {
             return true;
+        }
 
         return Boolean.parseBoolean(result);
     }

common/src/main/java/org/apache/rocketmq/common/protocol/NamespaceUtil.java

@@ -120,7 +120,7 @@ public class NamespaceUtil {
             return null;
         }
 
-        return new StringBuffer()
+        return new StringBuilder()
             .append(MixAll.RETRY_GROUP_TOPIC_PREFIX)
             .append(wrapNamespace(namespace, consumerGroup))
             .toString();

common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java

@@ -188,4 +188,6 @@ public class RequestCode {
     public static final int SEND_REPLY_MESSAGE_V2 = 325;
 
     public static final int PUSH_REPLY_MESSAGE_TO_CLIENT = 326;
+
+    public static final int ADD_WRITE_PERM_OF_BROKER = 327;
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/AddWritePermOfBrokerRequestHeader.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.protocol.header.namesrv;
+
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.annotation.CFNotNull;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+
+public class AddWritePermOfBrokerRequestHeader implements CommandCustomHeader {
+    @CFNotNull
+    private String brokerName;
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+
+    }
+
+    public String getBrokerName() {
+        return brokerName;
+    }
+
+    public void setBrokerName(String brokerName) {
+        this.brokerName = brokerName;
+    }
+}

common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/AddWritePermOfBrokerResponseHeader.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.common.protocol.header.namesrv;
+
+import org.apache.rocketmq.remoting.CommandCustomHeader;
+import org.apache.rocketmq.remoting.annotation.CFNotNull;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+
+public class AddWritePermOfBrokerResponseHeader implements CommandCustomHeader {
+    @CFNotNull
+    private Integer addTopicCount;
+
+    @Override
+    public void checkFields() throws RemotingCommandException {
+    }
+
+    public Integer getAddTopicCount() {
+        return addTopicCount;
+    }
+
+    public void setAddTopicCount(Integer addTopicCount) {
+        this.addTopicCount = addTopicCount;
+    }
+}

common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java

@@ -20,15 +20,16 @@ package org.apache.rocketmq.common.stats;
 import java.util.LinkedList;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
+
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.logging.InternalLogger;
 
 public class StatsItem {
 
-    private final AtomicLong value = new AtomicLong(0);
+    private final LongAdder value = new LongAdder();
 
-    private final AtomicLong times = new AtomicLong(0);
+    private final LongAdder times = new LongAdder();
 
     private final LinkedList<CallSnapshot> csListMinute = new LinkedList<CallSnapshot>();
 
@@ -157,8 +158,8 @@ public class StatsItem {
             if (this.csListMinute.size() == 0) {
                 this.csListMinute.add(new CallSnapshot(System.currentTimeMillis() - 10 * 1000, 0, 0));
             }
-            this.csListMinute.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value
-                .get()));
+            this.csListMinute.add(new CallSnapshot(System.currentTimeMillis(), this.times.sum(), this.value
+                .sum()));
             if (this.csListMinute.size() > 7) {
                 this.csListMinute.removeFirst();
             }
@@ -170,8 +171,8 @@ public class StatsItem {
             if (this.csListHour.size() == 0) {
                 this.csListHour.add(new CallSnapshot(System.currentTimeMillis() - 10 * 60 * 1000, 0, 0));
             }
-            this.csListHour.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value
-                .get()));
+            this.csListHour.add(new CallSnapshot(System.currentTimeMillis(), this.times.sum(), this.value
+                .sum()));
             if (this.csListHour.size() > 7) {
                 this.csListHour.removeFirst();
             }
@@ -183,8 +184,8 @@ public class StatsItem {
             if (this.csListDay.size() == 0) {
                 this.csListDay.add(new CallSnapshot(System.currentTimeMillis() - 1 * 60 * 60 * 1000, 0, 0));
             }
-            this.csListDay.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value
-                .get()));
+            this.csListDay.add(new CallSnapshot(System.currentTimeMillis(), this.times.sum(), this.value
+                .sum()));
             if (this.csListDay.size() > 25) {
                 this.csListDay.removeFirst();
             }
@@ -214,7 +215,7 @@ public class StatsItem {
                 ss.getAvgpt());
     }
 
-    public AtomicLong getValue() {
+    public LongAdder getValue() {
         return value;
     }
 
@@ -226,7 +227,7 @@ public class StatsItem {
         return statsName;
     }
 
-    public AtomicLong getTimes() {
+    public LongAdder getTimes() {
         return times;
     }
 }

common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java

@@ -154,14 +154,14 @@ public class StatsItemSet {
 
     public void addValue(final String statsKey, final int incValue, final int incTimes) {
         StatsItem statsItem = this.getAndCreateStatsItem(statsKey);
-        statsItem.getValue().addAndGet(incValue);
-        statsItem.getTimes().addAndGet(incTimes);
+        statsItem.getValue().add(incValue);
+        statsItem.getTimes().add(incTimes);
     }
 
     public void addRTValue(final String statsKey, final int incValue, final int incTimes) {
         StatsItem statsItem = this.getAndCreateRTStatsItem(statsKey);
-        statsItem.getValue().addAndGet(incValue);
-        statsItem.getTimes().addAndGet(incTimes);
+        statsItem.getValue().add(incValue);
+        statsItem.getTimes().add(incTimes);
     }
 
     public void delValue(final String statsKey) {

common/src/test/java/org/apache/rocketmq/common/RemotingUtilTest.java

@@ -28,4 +28,16 @@ public class RemotingUtilTest {
         assertThat(localAddress).isNotNull();
         assertThat(localAddress.length()).isGreaterThan(0);
     }
+
+    @Test
+    public void testConvert2IpStringWithIp() {
+        String result = RemotingUtil.convert2IpString("127.0.0.1:9876");
+        assertThat(result).isEqualTo("127.0.0.1:9876");
+    }
+
+    @Test
+    public void testConvert2IpStringWithHost() {
+        String result = RemotingUtil.convert2IpString("localhost:9876");
+        assertThat(result).isEqualTo("127.0.0.1:9876");
+    }
 }

common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java

@@ -19,11 +19,15 @@ package org.apache.rocketmq.common;
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.Properties;
 import org.junit.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.within;
+import static org.junit.Assert.assertEquals;
 
 public class UtilAllTest {
 
@@ -109,6 +113,15 @@ public class UtilAllTest {
         assertThat(UtilAll.ipToIPv6Str(nonInternal.getAddress()).toUpperCase()).isEqualTo("2408:4004:0180:8100:3FAA:1DDE:2B3F:898A");
     }
 
+    @Test
+    public void testJoin() {
+        List<String> list = Arrays.asList("groupA=DENY", "groupB=PUB|SUB", "groupC=SUB");
+        String comma = ",";
+        assertEquals("groupA=DENY,groupB=PUB|SUB,groupC=SUB", UtilAll.join(list, comma));
+        assertEquals(null, UtilAll.join(null, comma));
+        assertEquals("", UtilAll.join(Collections.emptyList(), comma));
+    }
+
     static class DemoConfig {
         private int demoWidth = 0;
         private int demoLength = 0;

common/src/test/java/org/apache/rocketmq/common/stats/StatsItemSetTest.java

@@ -23,6 +23,8 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
+
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.junit.After;
 import org.junit.Test;
@@ -95,7 +97,7 @@ public class StatsItemSetTest {
         }
     }
 
-    private AtomicLong test_unit() throws InterruptedException {
+    private LongAdder test_unit() throws InterruptedException {
         final StatsItemSet statsItemSet = new StatsItemSet("topicTest", scheduler, null);
         executor = new ThreadPoolExecutor(10, 20, 10, TimeUnit.SECONDS,
             new ArrayBlockingQueue<Runnable>(100), new ThreadFactoryImpl("testMultiThread"));

common/src/test/java/org/apache/rocketmq/common/utils/IOTinyUtilsTest.java

@@ -82,7 +82,7 @@ public class IOTinyUtilsTest {
 
     @Test
     public void testReadLines() throws Exception {
-        StringBuffer sb = new StringBuffer();
+        StringBuilder sb = new StringBuilder();
         for (int i = 0; i < 10; i++) {
             sb.append("testReadLines").append("\n");
         }
@@ -95,7 +95,7 @@ public class IOTinyUtilsTest {
 
     @Test
     public void testToBufferedReader() throws Exception {
-        StringBuffer sb = new StringBuffer();
+        StringBuilder sb = new StringBuilder();
         for (int i = 0; i < 10; i++) {
             sb.append("testToBufferedReader").append("\n");
         }

distribution/bin/export.sh

+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -z "$ROCKETMQ_HOME" ]; then
+  ## resolve links - $0 may be a link to maven's home
+  PRG="$0"
+
+  # need this for relative symlinks
+  while [ -h "$PRG" ]; do
+    ls=$(ls -ld "$PRG")
+    link=$(expr "$ls" : '.*-> \(.*\)$')
+    if expr "$link" : '/.*' >/dev/null; then
+      PRG="$link"
+    else
+      PRG="$(dirname "$PRG")/$link"
+    fi
+  done
+
+  saveddir=$(pwd)
+
+  ROCKETMQ_HOME=$(dirname "$PRG")/..
+
+  # make it fully qualified
+  ROCKETMQ_HOME=$(cd "$ROCKETMQ_HOME" && pwd)
+
+  cd "$saveddir"
+fi
+
+export ROCKETMQ_HOME
+
+namesrvAddr=
+while [ -z "${namesrvAddr}" ]; do
+  read -p "Enter name server address list:" namesrvAddr
+done
+
+clusterName=
+while [ -z "${clusterName}" ]; do
+  read -p "Choose a cluster to export:" clusterName
+done
+
+read -p "Enter file path to export [default /tmp/rocketmq/export]:" filePath
+if [ -z "${filePath}" ]; then
+  filePath="/tmp/rocketmq/config"
+fi
+
+if [[ -e ${filePath} ]]; then
+  rm -rf ${filePath}
+fi
+
+sh ${ROCKETMQ_HOME}/bin/mqadmin exportMetrics -c ${clusterName} -n ${namesrvAddr} -f ${filePath}
+sh ${ROCKETMQ_HOME}/bin/mqadmin exportConfigs -c ${clusterName} -n ${namesrvAddr} -f ${filePath}
+sh ${ROCKETMQ_HOME}/bin/mqadmin exportMetadata -c ${clusterName} -n ${namesrvAddr} -f ${filePath}
+
+cd ${filePath} || exit
+
+configs=$(cat ./configs.json)
+if [ -z "$configs" ]; then
+  configs="{}"
+fi
+metadata=$(cat ./metadata.json)
+if [ -z "$metadata" ]; then
+  metadata="{}"
+fi
+metrics=$(cat ./metrics.json)
+if [ -z "$metrics" ]; then
+  metrics="{}"
+fi
+
+echo "{
+    \"configs\": ${configs},
+    \"metadata\": ${metadata},
+    \"metrics\": ${metrics}
+  }" >rocketmq-metadata-export.json
+
+echo -e "[INFO] The RocketMQ metadata has been exported to the file:${filePath}/rocketmq-metadata-export.json"

distribution/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
     <artifactId>rocketmq-distribution</artifactId>
     <name>rocketmq-distribution ${project.version}</name>

docs/cn/RocketMQ_Example.md

@@ -54,7 +54,7 @@
 <dependency>
     <groupId>org.apache.rocketmq</groupId>
     <artifactId>rocketmq-client</artifactId>
-    <version>4.3.0</version>
+    <version>4.9.1</version>
 </dependency>
 ```
 `gradle`
@@ -120,13 +120,15 @@ public class AsyncProducer {
                 producer.send(msg, new SendCallback() {
                     @Override
                     public void onSuccess(SendResult sendResult) {
+                        countDownLatch.countDown();
                         System.out.printf("%-10d OK %s %n", index,
                             sendResult.getMsgId());
                     }
                     @Override
                     public void onException(Throwable e) {
-      	              System.out.printf("%-10d Exception %s %n", index, e);
-      	              e.printStackTrace();
+                        countDownLatch.countDown();
+      	                System.out.printf("%-10d Exception %s %n", index, e);
+      	                e.printStackTrace();
                     }
             	});
     	}
@@ -570,7 +572,7 @@ public class ListSplitter implements Iterator<List<Message>> {
         return currIndex; 
     }
     private int calcMessageSize(Message message) {
-        int tmpSize = message.getTopic().length() + message.getBody().length(); 
+        int tmpSize = message.getTopic().length() + message.getBody().length; 
         Map<String, String> properties = message.getProperties();
         for (Map.Entry<String, String> entry : properties.entrySet()) {
             tmpSize += entry.getKey().length() + entry.getValue().length(); 

docs/cn/operation.md

@@ -25,7 +25,7 @@ The Name Server boot success...
 ### 启动Broker
 $ nohup sh bin/mqbroker -n localhost:9876 &
 
-### 验证Name Server 是否启动成功，例如Broker的IP为：192.168.1.2，且名称为broker-a
+### 验证Broker是否启动成功，例如Broker的IP为：192.168.1.2，且名称为broker-a
 $ tail -f ~/logs/rocketmqlogs/broker.log 
 The broker[broker-a, 192.169.1.2:10911] boot success...
 ```
@@ -566,6 +566,14 @@ $ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker
   <td class=xl67 width=87 style='width:65pt'>-b</td>
   <td class=xl68 width=87 style='width:65pt'>BrokerName</td>
  </tr>
+ <tr height=57 style='height:43.0pt'>
+   <td rowspan=3 height=137 class=xl69 width=191 style='border-bottom:1.0pt;
+   height:103.0pt;border-top:none;width:143pt'>addWritePerm</td>
+   <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt
+   border-top:none;width:65pt'>从NameServer上添加 Broker写权限</td>
+   <td class=xl67 width=87 style='width:65pt'>-b</td>
+   <td class=xl68 width=87 style='width:65pt'>BrokerName</td>
+  </tr>
  <tr height=57 style='height:43.0pt'>
   <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>
   <td class=xl68 width=87 style='width:65pt'>NameServer 服务地址，格式 ip:port</td>

docs/en/CLITools.md

@@ -426,6 +426,14 @@ Before introducing the mqadmin management tool, the following points need to be
   <td class=xl67 width=87 style='width:65pt'>-b</td>
   <td class=xl68 width=87 style='width:65pt'>Declare the BrokerName</td>
  </tr>
+ <tr height=57 style='height:43.0pt'>
+   <td rowspan=3 height=137 class=xl69 width=191 style='border-bottom:1.0pt;
+   height:103.0pt;border-top:none;width:143pt'>addWritePerm</td>
+   <td rowspan=3 class=xl72 width=87 style='border-bottom:1.0pt
+   border-top:none;width:65pt'>Add write permissions for broker from nameServer</td>
+   <td class=xl67 width=87 style='width:65pt'>-b</td>
+   <td class=xl68 width=87 style='width:65pt'>Declare the BrokerName</td>
+  </tr>
  <tr height=57 style='height:43.0pt'>
   <td height=57 class=xl67 width=87 style='height:43.0pt;width:65pt'>-n</td>
   <td class=xl68 width=87 style='width:65pt'>Service address used to specify nameServer and formatted as ip:port</td>

docs/en/Example_Batch.md

@@ -57,7 +57,7 @@ public class ListSplitter implements Iterator<List<Message>> {
         return currIndex; 
     }
     private int calcMessageSize(Message message) {
-        int tmpSize = message.getTopic().length() + message.getBody().length(); 
+        int tmpSize = message.getTopic().length() + message.getBody().length; 
         Map<String, String> properties = message.getProperties();
         for (Map.Entry<String, String> entry : properties.entrySet()) {
             tmpSize += entry.getKey().length() + entry.getValue().length(); 
@@ -78,4 +78,4 @@ while (splitter.hasNext()) {
        // handle the error
    }
 }
-```
+```
\ No newline at end of file

docs/en/operation.md

@@ -25,7 +25,7 @@ The Name Server boot success...
 ### start Broker
 $ nohup sh bin/mqbroker -n localhost:9876 &
 
-### check whether Name Server is successfully started, eg: Broker's IP is 192.168.1.2, Broker's name is broker-a
+### check whether Broker is successfully started, eg: Broker's IP is 192.168.1.2, Broker's name is broker-a
 $ tail -f ~/logs/rocketmqlogs/Broker.log 
 The broker[broker-a, 192.169.1.2:10911] boot success...
 ```

example/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java

@@ -16,7 +16,7 @@
  */
 package org.apache.rocketmq.example.benchmark;
 
-import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
@@ -26,11 +26,14 @@ import java.util.TimerTask;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
+
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.PosixParser;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.rocketmq.acl.common.AclClientRPCHook;
 import org.apache.rocketmq.acl.common.SessionCredentials;
 import org.apache.rocketmq.client.exception.MQBrokerException;
@@ -42,13 +45,14 @@ import org.apache.rocketmq.client.producer.SendStatus;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.common.message.Message;
 import org.apache.rocketmq.remoting.RPCHook;
-import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.srvutil.ServerUtil;
 
 public class BatchProducer {
 
-    public static void main(String[] args) throws MQClientException, UnsupportedEncodingException {
+    private static byte[] msgBody;
+
+    public static void main(String[] args) throws MQClientException {
 
         Options options = ServerUtil.buildCommandlineOptions(new Options());
         CommandLine commandLine = ServerUtil.parseCmdLine("benchmarkBatchProducer", args, buildCommandlineOptions(options), new PosixParser());
@@ -72,6 +76,12 @@ public class BatchProducer {
         System.out.printf("topic: %s threadCount: %d messageSize: %d batchSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s%n",
                 topic, threadCount, messageSize, batchSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable);
 
+        StringBuilder sb = new StringBuilder(messageSize);
+        for (int i = 0; i < messageSize; i++) {
+            sb.append(RandomStringUtils.randomAlphanumeric(1));
+        }
+        msgBody = sb.toString().getBytes(StandardCharsets.UTF_8);
+
         final StatsBenchmarkBatchProducer statsBenchmark = new StatsBenchmarkBatchProducer();
         statsBenchmark.start();
 
@@ -85,14 +95,7 @@ public class BatchProducer {
                 @Override
                 public void run() {
                     while (true) {
-                        List<Message> msgs;
-
-                        try {
-                            msgs = buildBathMessage(batchSize, messageSize, topic);
-                        } catch (UnsupportedEncodingException e) {
-                            e.printStackTrace();
-                            return;
-                        }
+                        List<Message> msgs = buildBathMessage(batchSize, topic);
 
                         if (CollectionUtils.isEmpty(msgs)) {
                             return;
@@ -100,22 +103,22 @@ public class BatchProducer {
 
                         try {
                             long beginTimestamp = System.currentTimeMillis();
-                            long sendSucCount = statsBenchmark.getSendMessageSuccessCount().get();
+                            long sendSucCount = statsBenchmark.getSendMessageSuccessCount().longValue();
 
                             setKeys(keyEnable, msgs, String.valueOf(beginTimestamp / 1000));
                             setTags(tagCount, msgs, sendSucCount);
                             setProperties(propertySize, msgs);
                             SendResult sendResult = producer.send(msgs);
                             if (sendResult.getSendStatus() == SendStatus.SEND_OK) {
-                                statsBenchmark.getSendRequestSuccessCount().incrementAndGet();
-                                statsBenchmark.getSendMessageSuccessCount().addAndGet(msgs.size());
+                                statsBenchmark.getSendRequestSuccessCount().increment();
+                                statsBenchmark.getSendMessageSuccessCount().add(msgs.size());
                             } else {
-                                statsBenchmark.getSendRequestFailedCount().incrementAndGet();
-                                statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                                statsBenchmark.getSendRequestFailedCount().increment();
+                                statsBenchmark.getSendMessageFailedCount().add(msgs.size());
                             }
                             long currentRT = System.currentTimeMillis() - beginTimestamp;
-                            statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT);
-                            long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();
+                            statsBenchmark.getSendMessageSuccessTimeTotal().add(currentRT);
+                            long prevMaxRT = statsBenchmark.getSendMessageMaxRT().longValue();
                             while (currentRT > prevMaxRT) {
                                 boolean updated = statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT);
                                 if (updated) {
@@ -125,8 +128,8 @@ public class BatchProducer {
                                 prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();
                             }
                         } catch (RemotingException e) {
-                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
-                            statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            statsBenchmark.getSendRequestFailedCount().increment();
+                            statsBenchmark.getSendMessageFailedCount().add(msgs.size());
                             log.error("[BENCHMARK_PRODUCER] Send Exception", e);
 
                             try {
@@ -134,22 +137,22 @@ public class BatchProducer {
                             } catch (InterruptedException ignored) {
                             }
                         } catch (InterruptedException e) {
-                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
-                            statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            statsBenchmark.getSendRequestFailedCount().increment();
+                            statsBenchmark.getSendMessageFailedCount().add(msgs.size());
                             try {
                                 Thread.sleep(3000);
                             } catch (InterruptedException e1) {
                             }
-                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
-                            statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            statsBenchmark.getSendRequestFailedCount().increment();
+                            statsBenchmark.getSendMessageFailedCount().add(msgs.size());
                             log.error("[BENCHMARK_PRODUCER] Send Exception", e);
                         } catch (MQClientException e) {
-                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
-                            statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            statsBenchmark.getSendRequestFailedCount().increment();
+                            statsBenchmark.getSendMessageFailedCount().add(msgs.size());
                             log.error("[BENCHMARK_PRODUCER] Send Exception", e);
                         } catch (MQBrokerException e) {
-                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
-                            statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size());
+                            statsBenchmark.getSendRequestFailedCount().increment();
+                            statsBenchmark.getSendMessageFailedCount().add(msgs.size());
                             log.error("[BENCHMARK_PRODUCER] Send Exception", e);
                             try {
                                 Thread.sleep(3000);
@@ -234,23 +237,12 @@ public class BatchProducer {
         return defaultValue;
     }
 
-    private static List<Message> buildBathMessage(int batchSize, int messageSize,
-                                                  String topic) throws UnsupportedEncodingException {
+    private static List<Message> buildBathMessage(final int batchSize, final String topic) {
         List<Message> batchMessage = new ArrayList<>(batchSize);
-
         for (int i = 0; i < batchSize; i++) {
-            Message msg = new Message();
-            msg.setTopic(topic);
-
-            StringBuilder sb = new StringBuilder();
-            for (int j = 0; j < messageSize; j += 10) {
-                sb.append("hello baby");
-            }
-
-            msg.setBody(sb.toString().getBytes(RemotingHelper.DEFAULT_CHARSET));
+            Message msg = new Message(topic, msgBody);
             batchMessage.add(msg);
         }
-
         return batchMessage;
     }
 
@@ -313,17 +305,17 @@ public class BatchProducer {
 
 class StatsBenchmarkBatchProducer {
 
-    private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L);
+    private final LongAdder sendRequestSuccessCount = new LongAdder();
 
-    private final AtomicLong sendRequestFailedCount = new AtomicLong(0L);
+    private final LongAdder sendRequestFailedCount = new LongAdder();
 
-    private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L);
+    private final LongAdder sendMessageSuccessTimeTotal = new LongAdder();
 
     private final AtomicLong sendMessageMaxRT = new AtomicLong(0L);
 
-    private final AtomicLong sendMessageSuccessCount = new AtomicLong(0L);
+    private final LongAdder sendMessageSuccessCount = new LongAdder();
 
-    private final AtomicLong sendMessageFailedCount = new AtomicLong(0L);
+    private final LongAdder sendMessageFailedCount = new LongAdder();
 
     private final Timer timer = new Timer("BenchmarkTimerThread", true);
 
@@ -332,25 +324,25 @@ class StatsBenchmarkBatchProducer {
     public Long[] createSnapshot() {
         Long[] snap = new Long[] {
                 System.currentTimeMillis(),
-                this.sendRequestSuccessCount.get(),
-                this.sendRequestFailedCount.get(),
-                this.sendMessageSuccessCount.get(),
-                this.sendMessageFailedCount.get(),
-                this.sendMessageSuccessTimeTotal.get(),
+                this.sendRequestSuccessCount.longValue(),
+                this.sendRequestFailedCount.longValue(),
+                this.sendMessageSuccessCount.longValue(),
+                this.sendMessageFailedCount.longValue(),
+                this.sendMessageSuccessTimeTotal.longValue(),
         };
 
         return snap;
     }
 
-    public AtomicLong getSendRequestSuccessCount() {
+    public LongAdder getSendRequestSuccessCount() {
         return sendRequestSuccessCount;
     }
 
-    public AtomicLong getSendRequestFailedCount() {
+    public LongAdder getSendRequestFailedCount() {
         return sendRequestFailedCount;
     }
 
-    public AtomicLong getSendMessageSuccessTimeTotal() {
+    public LongAdder getSendMessageSuccessTimeTotal() {
         return sendMessageSuccessTimeTotal;
     }
 
@@ -358,11 +350,11 @@ class StatsBenchmarkBatchProducer {
         return sendMessageMaxRT;
     }
 
-    public AtomicLong getSendMessageSuccessCount() {
+    public LongAdder getSendMessageSuccessCount() {
         return sendMessageSuccessCount;
     }
 
-    public AtomicLong getSendMessageFailedCount() {
+    public LongAdder getSendMessageFailedCount() {
         return sendMessageFailedCount;
     }
 
@@ -390,7 +382,7 @@ class StatsBenchmarkBatchProducer {
                     final double averageMsgRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]);
 
                     System.out.printf("Current Time: %s Send TPS: %d Send MPS: %d Max RT(ms): %d Average RT(ms): %7.3f Average Message RT(ms): %7.3f Send Failed: %d Send Message Failed: %d%n",
-                            System.currentTimeMillis(), sendTps, sendMps, getSendMessageMaxRT().get(), averageRT, averageMsgRT, end[2], end[4]);
+                            System.currentTimeMillis(), sendTps, sendMps, getSendMessageMaxRT().longValue(), averageRT, averageMsgRT, end[2], end[4]);
                 }
             }
 

example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java

@@ -17,18 +17,12 @@
 
 package org.apache.rocketmq.example.benchmark;
 
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicLong;
-
+import java.util.concurrent.atomic.LongAdder;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
 import org.apache.rocketmq.client.consumer.MessageSelector;
 import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
@@ -42,6 +36,16 @@ import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.srvutil.ServerUtil;
 
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.TimerTask;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
 public class Consumer {
 
     public static void main(String[] args) throws MQClientException, IOException {
@@ -71,11 +75,12 @@ public class Consumer {
 
         final StatsBenchmarkConsumer statsBenchmarkConsumer = new StatsBenchmarkConsumer();
 
-        final Timer timer = new Timer("BenchmarkTimerThread", true);
+        ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1,
+                new BasicThreadFactory.Builder().namingPattern("BenchmarkTimerThread-%d").daemon(true).build());
 
         final LinkedList<Long[]> snapshotList = new LinkedList<Long[]>();
 
-        timer.scheduleAtFixedRate(new TimerTask() {
+        executorService.scheduleAtFixedRate(new TimerTask() {
             @Override
             public void run() {
                 snapshotList.addLast(statsBenchmarkConsumer.createSnapshot());
@@ -83,9 +88,9 @@ public class Consumer {
                     snapshotList.removeFirst();
                 }
             }
-        }, 1000, 1000);
+        }, 1000, 1000, TimeUnit.MILLISECONDS);
 
-        timer.scheduleAtFixedRate(new TimerTask() {
+        executorService.scheduleAtFixedRate(new TimerTask() {
             private void printStats() {
                 if (snapshotList.size() >= 10) {
                     Long[] begin = snapshotList.getFirst();
@@ -116,7 +121,7 @@ public class Consumer {
                     e.printStackTrace();
                 }
             }
-        }, 10000, 10000);
+        }, 10000, 10000, TimeUnit.MILLISECONDS);
 
         RPCHook rpcHook = aclEnable ? AclClient.getAclRPCHook() : null;
         DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group, rpcHook, new AllocateMessageQueueAveragely(), msgTraceEnable, null);
@@ -151,20 +156,20 @@ public class Consumer {
                 MessageExt msg = msgs.get(0);
                 long now = System.currentTimeMillis();
 
-                statsBenchmarkConsumer.getReceiveMessageTotalCount().incrementAndGet();
+                statsBenchmarkConsumer.getReceiveMessageTotalCount().increment();
 
                 long born2ConsumerRT = now - msg.getBornTimestamp();
-                statsBenchmarkConsumer.getBorn2ConsumerTotalRT().addAndGet(born2ConsumerRT);
+                statsBenchmarkConsumer.getBorn2ConsumerTotalRT().add(born2ConsumerRT);
 
                 long store2ConsumerRT = now - msg.getStoreTimestamp();
-                statsBenchmarkConsumer.getStore2ConsumerTotalRT().addAndGet(store2ConsumerRT);
+                statsBenchmarkConsumer.getStore2ConsumerTotalRT().add(store2ConsumerRT);
 
                 compareAndSetMax(statsBenchmarkConsumer.getBorn2ConsumerMaxRT(), born2ConsumerRT);
 
                 compareAndSetMax(statsBenchmarkConsumer.getStore2ConsumerMaxRT(), store2ConsumerRT);
 
                 if (ThreadLocalRandom.current().nextDouble() < failRate) {
-                    statsBenchmarkConsumer.getFailCount().incrementAndGet();
+                    statsBenchmarkConsumer.getFailCount().increment();
                     return ConsumeConcurrentlyStatus.RECONSUME_LATER;
                 } else {
                     return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
@@ -229,39 +234,39 @@ public class Consumer {
 }
 
 class StatsBenchmarkConsumer {
-    private final AtomicLong receiveMessageTotalCount = new AtomicLong(0L);
+    private final LongAdder receiveMessageTotalCount = new LongAdder();
 
-    private final AtomicLong born2ConsumerTotalRT = new AtomicLong(0L);
+    private final LongAdder born2ConsumerTotalRT = new LongAdder();
 
-    private final AtomicLong store2ConsumerTotalRT = new AtomicLong(0L);
+    private final LongAdder store2ConsumerTotalRT = new LongAdder();
 
     private final AtomicLong born2ConsumerMaxRT = new AtomicLong(0L);
 
     private final AtomicLong store2ConsumerMaxRT = new AtomicLong(0L);
 
-    private final AtomicLong failCount = new AtomicLong(0L);
+    private final LongAdder failCount = new LongAdder();
 
     public Long[] createSnapshot() {
         Long[] snap = new Long[] {
             System.currentTimeMillis(),
-            this.receiveMessageTotalCount.get(),
-            this.born2ConsumerTotalRT.get(),
-            this.store2ConsumerTotalRT.get(),
-            this.failCount.get()
+            this.receiveMessageTotalCount.longValue(),
+            this.born2ConsumerTotalRT.longValue(),
+            this.store2ConsumerTotalRT.longValue(),
+            this.failCount.longValue()
         };
 
         return snap;
     }
 
-    public AtomicLong getReceiveMessageTotalCount() {
+    public LongAdder getReceiveMessageTotalCount() {
         return receiveMessageTotalCount;
     }
 
-    public AtomicLong getBorn2ConsumerTotalRT() {
+    public LongAdder getBorn2ConsumerTotalRT() {
         return born2ConsumerTotalRT;
     }
 
-    public AtomicLong getStore2ConsumerTotalRT() {
+    public LongAdder getStore2ConsumerTotalRT() {
         return store2ConsumerTotalRT;
     }
 
@@ -273,7 +278,7 @@ class StatsBenchmarkConsumer {
         return store2ConsumerMaxRT;
     }
 
-    public AtomicLong getFailCount() {
+    public LongAdder getFailCount() {
         return failCount;
     }
 }

example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java

@@ -16,35 +16,40 @@
  */
 package org.apache.rocketmq.example.benchmark;
 
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.Random;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.atomic.LongAdder;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.log.ClientLogger;
 import org.apache.rocketmq.client.producer.DefaultMQProducer;
-import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.remoting.RPCHook;
-import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.srvutil.ServerUtil;
 
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.Random;
+import java.util.TimerTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicLong;
+
 public class Producer {
 
-    public static void main(String[] args) throws MQClientException, UnsupportedEncodingException {
+    private static byte[] msgBody;
+
+    public static void main(String[] args) throws MQClientException {
 
         Options options = ServerUtil.buildCommandlineOptions(new Options());
         CommandLine commandLine = ServerUtil.parseCmdLine("benchmarkProducer", args, buildCommandlineOptions(options), new PosixParser());
@@ -67,13 +72,20 @@ public class Producer {
         System.out.printf("topic: %s threadCount: %d messageSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s messageQuantity: %d%n delayEnable: %s%n delayLevel: %s%n",
             topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable, messageNum, delayEnable, delayLevel);
 
+        StringBuilder sb = new StringBuilder(messageSize);
+        for (int i = 0; i < messageSize; i++) {
+            sb.append(RandomStringUtils.randomAlphanumeric(1));
+        }
+        msgBody = sb.toString().getBytes(StandardCharsets.UTF_8);
+
         final InternalLogger log = ClientLogger.getLog();
 
         final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount);
 
         final StatsBenchmarkProducer statsBenchmark = new StatsBenchmarkProducer();
 
-        final Timer timer = new Timer("BenchmarkTimerThread", true);
+        ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1,
+                new BasicThreadFactory.Builder().namingPattern("BenchmarkTimerThread-%d").daemon(true).build());
 
         final LinkedList<Long[]> snapshotList = new LinkedList<Long[]>();
 
@@ -87,7 +99,7 @@ public class Producer {
             }
         }
 
-        timer.scheduleAtFixedRate(new TimerTask() {
+        executorService.scheduleAtFixedRate(new TimerTask() {
             @Override
             public void run() {
                 snapshotList.addLast(statsBenchmark.createSnapshot());
@@ -95,9 +107,9 @@ public class Producer {
                     snapshotList.removeFirst();
                 }
             }
-        }, 1000, 1000);
+        }, 1000, 1000, TimeUnit.MILLISECONDS);
 
-        timer.scheduleAtFixedRate(new TimerTask() {
+        executorService.scheduleAtFixedRate(new TimerTask() {
             private void printStats() {
                 if (snapshotList.size() >= 10) {
                     doPrintStats(snapshotList,  statsBenchmark, false);
@@ -112,7 +124,7 @@ public class Producer {
                     e.printStackTrace();
                 }
             }
-        }, 10000, 10000);
+        }, 10000, 10000, TimeUnit.MILLISECONDS);
 
         RPCHook rpcHook = aclEnable ? AclClient.getAclRPCHook() : null;
         final DefaultMQProducer producer = new DefaultMQProducer("benchmark_producer", rpcHook, msgTraceEnable, null);
@@ -138,13 +150,7 @@ public class Producer {
                     int num = 0;
                     while (true) {
                         try {
-                            final Message msg;
-                            try {
-                                msg = buildMessage(messageSize, topic);
-                            } catch (UnsupportedEncodingException e) {
-                                e.printStackTrace();
-                                return;
-                            }
+                            final Message msg = buildMessage(topic);
                             final long beginTimestamp = System.currentTimeMillis();
                             if (keyEnable) {
                                 msg.setKeys(String.valueOf(beginTimestamp / 1000));
@@ -153,8 +159,7 @@ public class Producer {
                                 msg.setDelayTimeLevel(delayLevel);
                             }
                             if (tagCount > 0) {
-                                long sendSucCount = statsBenchmark.getReceiveResponseSuccessCount().get();
-                                msg.setTags(String.format("tag%d", sendSucCount % tagCount));
+                                msg.setTags(String.format("tag%d", System.currentTimeMillis() % tagCount));
                             }
                             if (propertySize > 0) {
                                 if (msg.getProperties() != null) {
@@ -177,20 +182,20 @@ public class Producer {
                                 }
                             }
                             producer.send(msg);
-                            statsBenchmark.getSendRequestSuccessCount().incrementAndGet();
-                            statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet();
+                            statsBenchmark.getSendRequestSuccessCount().increment();
+                            statsBenchmark.getReceiveResponseSuccessCount().increment();
                             final long currentRT = System.currentTimeMillis() - beginTimestamp;
-                            statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT);
-                            long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();
+                            statsBenchmark.getSendMessageSuccessTimeTotal().add(currentRT);
+                            long prevMaxRT = statsBenchmark.getSendMessageMaxRT().longValue();
                             while (currentRT > prevMaxRT) {
                                 boolean updated = statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT);
                                 if (updated)
                                     break;
 
-                                prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();
+                                prevMaxRT = statsBenchmark.getSendMessageMaxRT().longValue();
                             }
                         } catch (RemotingException e) {
-                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+                            statsBenchmark.getSendRequestFailedCount().increment();
                             log.error("[BENCHMARK_PRODUCER] Send Exception", e);
 
                             try {
@@ -198,16 +203,16 @@ public class Producer {
                             } catch (InterruptedException ignored) {
                             }
                         } catch (InterruptedException e) {
-                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+                            statsBenchmark.getSendRequestFailedCount().increment();
                             try {
                                 Thread.sleep(3000);
                             } catch (InterruptedException e1) {
                             }
                         } catch (MQClientException e) {
-                            statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+                            statsBenchmark.getSendRequestFailedCount().increment();
                             log.error("[BENCHMARK_PRODUCER] Send Exception", e);
                         } catch (MQBrokerException e) {
-                            statsBenchmark.getReceiveResponseFailedCount().incrementAndGet();
+                            statsBenchmark.getReceiveResponseFailedCount().increment();
                             log.error("[BENCHMARK_PRODUCER] Send Exception", e);
                             try {
                                 Thread.sleep(3000);
@@ -224,13 +229,18 @@ public class Producer {
         try {
             sendThreadPool.shutdown();
             sendThreadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
-            timer.cancel();
+            executorService.shutdown();
+            try {
+                executorService.awaitTermination(5000, TimeUnit.MILLISECONDS);
+            } catch (InterruptedException e) {
+            }
+
             if (snapshotList.size() > 1) {
                 doPrintStats(snapshotList, statsBenchmark, true);
             } else {
                 System.out.printf("[Complete] Send Total: %d Send Failed: %d Response Failed: %d%n",
-                    statsBenchmark.getSendRequestSuccessCount().get() + statsBenchmark.getSendRequestFailedCount().get(),
-                    statsBenchmark.getSendRequestFailedCount().get(), statsBenchmark.getReceiveResponseFailedCount().get());
+                    statsBenchmark.getSendRequestSuccessCount().longValue() + statsBenchmark.getSendRequestFailedCount().longValue(),
+                    statsBenchmark.getSendRequestFailedCount().longValue(), statsBenchmark.getReceiveResponseFailedCount().longValue());
             }
             producer.shutdown();
         } catch (InterruptedException e) {
@@ -282,18 +292,8 @@ public class Producer {
         return options;
     }
 
-    private static Message buildMessage(final int messageSize, final String topic) throws UnsupportedEncodingException {
-        Message msg = new Message();
-        msg.setTopic(topic);
-
-        StringBuilder sb = new StringBuilder();
-        for (int i = 0; i < messageSize; i += 10) {
-            sb.append("hello baby");
-        }
-
-        msg.setBody(sb.toString().getBytes(RemotingHelper.DEFAULT_CHARSET));
-
-        return msg;
+    private static Message buildMessage(final String topic) {
+        return new Message(topic, msgBody);
     }
 
     private static void doPrintStats(final LinkedList<Long[]> snapshotList, final StatsBenchmarkProducer statsBenchmark, boolean done) {
@@ -305,58 +305,58 @@ public class Producer {
 
         if (done) {
             System.out.printf("[Complete] Send Total: %d Send TPS: %d Max RT(ms): %d Average RT(ms): %7.3f Send Failed: %d Response Failed: %d%n",
-                statsBenchmark.getSendRequestSuccessCount().get() + statsBenchmark.getSendRequestFailedCount().get(),
-                sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]);
+                statsBenchmark.getSendRequestSuccessCount().longValue() + statsBenchmark.getSendRequestFailedCount().longValue(),
+                sendTps, statsBenchmark.getSendMessageMaxRT().longValue(), averageRT, end[2], end[4]);
         } else {
             System.out.printf("Current Time: %s Send TPS: %d Max RT(ms): %d Average RT(ms): %7.3f Send Failed: %d Response Failed: %d%n",
-                System.currentTimeMillis(), sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]);
+                System.currentTimeMillis(), sendTps, statsBenchmark.getSendMessageMaxRT().longValue(), averageRT, end[2], end[4]);
         }
     }
 }
 
 class StatsBenchmarkProducer {
-    private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L);
+    private final LongAdder sendRequestSuccessCount = new LongAdder();
 
-    private final AtomicLong sendRequestFailedCount = new AtomicLong(0L);
+    private final LongAdder sendRequestFailedCount = new LongAdder();
 
-    private final AtomicLong receiveResponseSuccessCount = new AtomicLong(0L);
+    private final LongAdder receiveResponseSuccessCount = new LongAdder();
 
-    private final AtomicLong receiveResponseFailedCount = new AtomicLong(0L);
+    private final LongAdder receiveResponseFailedCount = new LongAdder();
 
-    private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L);
+    private final LongAdder sendMessageSuccessTimeTotal = new LongAdder();
 
     private final AtomicLong sendMessageMaxRT = new AtomicLong(0L);
 
     public Long[] createSnapshot() {
         Long[] snap = new Long[] {
             System.currentTimeMillis(),
-            this.sendRequestSuccessCount.get(),
-            this.sendRequestFailedCount.get(),
-            this.receiveResponseSuccessCount.get(),
-            this.receiveResponseFailedCount.get(),
-            this.sendMessageSuccessTimeTotal.get(),
+            this.sendRequestSuccessCount.longValue(),
+            this.sendRequestFailedCount.longValue(),
+            this.receiveResponseSuccessCount.longValue(),
+            this.receiveResponseFailedCount.longValue(),
+            this.sendMessageSuccessTimeTotal.longValue(),
         };
 
         return snap;
     }
 
-    public AtomicLong getSendRequestSuccessCount() {
+    public LongAdder getSendRequestSuccessCount() {
         return sendRequestSuccessCount;
     }
 
-    public AtomicLong getSendRequestFailedCount() {
+    public LongAdder getSendRequestFailedCount() {
         return sendRequestFailedCount;
     }
 
-    public AtomicLong getReceiveResponseSuccessCount() {
+    public LongAdder getReceiveResponseSuccessCount() {
         return receiveResponseSuccessCount;
     }
 
-    public AtomicLong getReceiveResponseFailedCount() {
+    public LongAdder getReceiveResponseFailedCount() {
         return receiveResponseFailedCount;
     }
 
-    public AtomicLong getSendMessageSuccessTimeTotal() {
+    public LongAdder getSendMessageSuccessTimeTotal() {
         return sendMessageSuccessTimeTotal;
     }
 

example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java

@@ -17,26 +17,11 @@
 
 package org.apache.rocketmq.example.benchmark;
 
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicLong;
-
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.producer.LocalTransactionState;
 import org.apache.rocketmq.client.producer.SendResult;
@@ -48,9 +33,28 @@ import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.srvutil.ServerUtil;
 
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.TimerTask;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.LongAdder;
+
 public class TransactionProducer {
     private static final long START_TIME = System.currentTimeMillis();
-    private static final AtomicLong MSG_COUNT = new AtomicLong(0);
+    private static final LongAdder MSG_COUNT = new LongAdder();
 
     //broker max check times should less than this value
     static final int MAX_CHECK_RESULT_IN_MSG = 20;
@@ -75,11 +79,12 @@ public class TransactionProducer {
 
         final StatsBenchmarkTProducer statsBenchmark = new StatsBenchmarkTProducer();
 
-        final Timer timer = new Timer("BenchmarkTimerThread", true);
+        ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1,
+                new BasicThreadFactory.Builder().namingPattern("BenchmarkTimerThread-%d").daemon(true).build());
 
         final LinkedList<Snapshot> snapshotList = new LinkedList<>();
 
-        timer.scheduleAtFixedRate(new TimerTask() {
+        executorService.scheduleAtFixedRate(new TimerTask() {
             @Override
             public void run() {
                 snapshotList.addLast(statsBenchmark.createSnapshot());
@@ -87,9 +92,9 @@ public class TransactionProducer {
                     snapshotList.removeFirst();
                 }
             }
-        }, 1000, 1000);
+        }, 1000, 1000, TimeUnit.MILLISECONDS);
 
-        timer.scheduleAtFixedRate(new TimerTask() {
+        executorService.scheduleAtFixedRate(new TimerTask() {
             private void printStats() {
                 if (snapshotList.size() >= 10) {
                     Snapshot begin = snapshotList.getFirst();
@@ -121,7 +126,7 @@ public class TransactionProducer {
                     e.printStackTrace();
                 }
             }
-        }, 10000, 10000);
+        }, 10000, 10000, TimeUnit.MILLISECONDS);
 
         final TransactionListener transactionCheckListener = new TransactionListenerImpl(statsBenchmark, config);
         final TransactionMQProducer producer = new TransactionMQProducer(
@@ -154,7 +159,7 @@ public class TransactionProducer {
                             success = false;
                         } finally {
                             final long currentRT = System.currentTimeMillis() - beginTimestamp;
-                            statsBenchmark.getSendMessageTimeTotal().addAndGet(currentRT);
+                            statsBenchmark.getSendMessageTimeTotal().add(currentRT);
                             long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();
                             while (currentRT > prevMaxRT) {
                                 boolean updated = statsBenchmark.getSendMessageMaxRT()
@@ -165,9 +170,9 @@ public class TransactionProducer {
                                 prevMaxRT = statsBenchmark.getSendMessageMaxRT().get();
                             }
                             if (success) {
-                                statsBenchmark.getSendRequestSuccessCount().incrementAndGet();
+                                statsBenchmark.getSendRequestSuccessCount().increment();
                             } else {
-                                statsBenchmark.getSendRequestFailedCount().incrementAndGet();
+                                statsBenchmark.getSendRequestFailedCount().increment();
                             }
                             if (config.sendInterval > 0) {
                                 try {
@@ -190,7 +195,9 @@ public class TransactionProducer {
         ByteBuffer buf = ByteBuffer.wrap(bs);
         buf.putLong(config.batchId);
         long sendMachineId = START_TIME << 32;
-        long msgId = sendMachineId | MSG_COUNT.getAndIncrement();
+        long count = MSG_COUNT.longValue();
+        long msgId = sendMachineId | count;
+        MSG_COUNT.increment();
         buf.putLong(msgId);
 
         // save send tx result in message
@@ -312,7 +319,7 @@ class TransactionListenerImpl implements TransactionListener {
             // message not generated in this test
             return LocalTransactionState.ROLLBACK_MESSAGE;
         }
-        statBenchmark.getCheckCount().incrementAndGet();
+        statBenchmark.getCheckCount().increment();
 
         int times = 0;
         try {
@@ -335,7 +342,7 @@ class TransactionListenerImpl implements TransactionListener {
             dup = newCheckLog.equals(oldCheckLog);
         }
         if (dup) {
-            statBenchmark.getDuplicatedCheckCount().incrementAndGet();
+            statBenchmark.getDuplicatedCheckCount().increment();
         }
         if (msgMeta.sendResult != LocalTransactionState.UNKNOW) {
             System.out.printf("%s unexpected check: msgId=%s,txId=%s,checkTimes=%s,sendResult=%s\n",
@@ -343,7 +350,7 @@ class TransactionListenerImpl implements TransactionListener {
                     msg.getMsgId(), msg.getTransactionId(),
                     msg.getUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES),
                     msgMeta.sendResult.toString());
-            statBenchmark.getUnexpectedCheckCount().incrementAndGet();
+            statBenchmark.getUnexpectedCheckCount().increment();
             return msgMeta.sendResult;
         }
 
@@ -354,7 +361,7 @@ class TransactionListenerImpl implements TransactionListener {
                         new SimpleDateFormat("HH:mm:ss,SSS").format(new Date()),
                         msg.getMsgId(), msg.getTransactionId(),
                         msg.getUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES), s);
-                statBenchmark.getUnexpectedCheckCount().incrementAndGet();
+                statBenchmark.getUnexpectedCheckCount().increment();
                 return s;
             }
         }
@@ -381,42 +388,42 @@ class Snapshot {
 }
 
 class StatsBenchmarkTProducer {
-    private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L);
+    private final LongAdder sendRequestSuccessCount = new LongAdder();
 
-    private final AtomicLong sendRequestFailedCount = new AtomicLong(0L);
+    private final LongAdder sendRequestFailedCount = new LongAdder();
 
-    private final AtomicLong sendMessageTimeTotal = new AtomicLong(0L);
+    private final LongAdder sendMessageTimeTotal = new LongAdder();
 
     private final AtomicLong sendMessageMaxRT = new AtomicLong(0L);
 
-    private final AtomicLong checkCount = new AtomicLong(0L);
+    private final LongAdder checkCount = new LongAdder();
 
-    private final AtomicLong unexpectedCheckCount = new AtomicLong(0L);
+    private final LongAdder unexpectedCheckCount = new LongAdder();
 
-    private final AtomicLong duplicatedCheckCount = new AtomicLong(0);
+    private final LongAdder duplicatedCheckCount = new LongAdder();
 
     public Snapshot createSnapshot() {
         Snapshot s = new Snapshot();
         s.endTime = System.currentTimeMillis();
-        s.sendRequestSuccessCount = sendRequestSuccessCount.get();
-        s.sendRequestFailedCount = sendRequestFailedCount.get();
-        s.sendMessageTimeTotal = sendMessageTimeTotal.get();
+        s.sendRequestSuccessCount = sendRequestSuccessCount.longValue();
+        s.sendRequestFailedCount = sendRequestFailedCount.longValue();
+        s.sendMessageTimeTotal = sendMessageTimeTotal.longValue();
         s.sendMessageMaxRT = sendMessageMaxRT.get();
-        s.checkCount = checkCount.get();
-        s.unexpectedCheckCount = unexpectedCheckCount.get();
-        s.duplicatedCheck = duplicatedCheckCount.get();
+        s.checkCount = checkCount.longValue();
+        s.unexpectedCheckCount = unexpectedCheckCount.longValue();
+        s.duplicatedCheck = duplicatedCheckCount.longValue();
         return s;
     }
 
-    public AtomicLong getSendRequestSuccessCount() {
+    public LongAdder getSendRequestSuccessCount() {
         return sendRequestSuccessCount;
     }
 
-    public AtomicLong getSendRequestFailedCount() {
+    public LongAdder getSendRequestFailedCount() {
         return sendRequestFailedCount;
     }
 
-    public AtomicLong getSendMessageTimeTotal() {
+    public LongAdder getSendMessageTimeTotal() {
         return sendMessageTimeTotal;
     }
 
@@ -424,15 +431,15 @@ class StatsBenchmarkTProducer {
         return sendMessageMaxRT;
     }
 
-    public AtomicLong getCheckCount() {
+    public LongAdder getCheckCount() {
         return checkCount;
     }
 
-    public AtomicLong getUnexpectedCheckCount() {
+    public LongAdder getUnexpectedCheckCount() {
         return unexpectedCheckCount;
     }
 
-    public AtomicLong getDuplicatedCheckCount() {
+    public LongAdder getDuplicatedCheckCount() {
         return duplicatedCheckCount;
     }
 }

example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java

@@ -16,63 +16,133 @@
  */
 package org.apache.rocketmq.example.simple;
 
-import java.util.HashMap;
-import java.util.Map;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+
 import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
 import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.store.ReadOffsetType;
+import org.apache.rocketmq.client.exception.MQBrokerException;
 import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.exception.RemotingException;
 
+@SuppressWarnings("deprecation")
 public class PullConsumer {
-    private static final Map<MessageQueue, Long> OFFSE_TABLE = new HashMap<MessageQueue, Long>();
 
     public static void main(String[] args) throws MQClientException {
+
         DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_5");
         consumer.setNamesrvAddr("127.0.0.1:9876");
+        Set<String> topics = new HashSet<>();
+        //You would better to register topics,It will use in rebalance when starting
+        topics.add("TopicTest");
+        consumer.setRegisterTopics(topics);
         consumer.start();
 
-        Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues("broker-a");
-        for (MessageQueue mq : mqs) {
-            System.out.printf("Consume from the queue: %s%n", mq);
-            SINGLE_MQ:
-            while (true) {
-                try {
-                    PullResult pullResult =
-                        consumer.pullBlockIfNotFound(mq, null, getMessageQueueOffset(mq), 32);
-                    System.out.printf("%s%n", pullResult);
-                    putMessageQueueOffset(mq, pullResult.getNextBeginOffset());
-                    switch (pullResult.getPullStatus()) {
-                        case FOUND:
-                            break;
-                        case NO_MATCHED_MSG:
-                            break;
-                        case NO_NEW_MSG:
-                            break SINGLE_MQ;
-                        case OFFSET_ILLEGAL:
-                            break;
-                        default:
-                            break;
-                    }
-                } catch (Exception e) {
-                    e.printStackTrace();
-                }
+        ExecutorService executors = Executors.newFixedThreadPool(topics.size(), new ThreadFactory() {
+            @Override
+            public Thread newThread(Runnable r) {
+                return new Thread(r, "PullConsumerThread");
             }
-        }
+        });
+        for (String topic : consumer.getRegisterTopics()) {
 
-        consumer.shutdown();
-    }
+            executors.execute(new Runnable() {
 
-    private static long getMessageQueueOffset(MessageQueue mq) {
-        Long offset = OFFSE_TABLE.get(mq);
-        if (offset != null)
-            return offset;
+                public void doSomething(List<MessageExt> msgs) {
+                    //do you business
 
-        return 0;
-    }
+                }
 
-    private static void putMessageQueueOffset(MessageQueue mq, long offset) {
-        OFFSE_TABLE.put(mq, offset);
-    }
+                @Override
+                public void run() {
+                    while (true) {
+                        try {
+                            Set<MessageQueue> messageQueues = consumer.fetchMessageQueuesInBalance(topic);
+                            if (messageQueues == null || messageQueues.isEmpty()) {
+                                Thread.sleep(1000);
+                                continue;
+                            }
+                            PullResult pullResult = null;
+                            for (MessageQueue messageQueue : messageQueues) {
+                                try {
+                                    long offset = this.consumeFromOffset(messageQueue);
+                                    pullResult = consumer.pull(messageQueue, "*", offset, 32);
+                                    switch (pullResult.getPullStatus()) {
+                                        case FOUND:
+                                            List<MessageExt> msgs = pullResult.getMsgFoundList();
 
+                                            if (msgs != null && !msgs.isEmpty()) {
+                                                this.doSomething(msgs);
+                                                //update offset to broker
+                                                consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());
+                                                //print pull tps
+                                                this.incPullTPS(topic, pullResult.getMsgFoundList().size());
+                                            }
+                                            break;
+                                        case OFFSET_ILLEGAL:
+                                            consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());
+                                            break;
+                                        case NO_NEW_MSG:
+                                            Thread.sleep(1);
+                                            consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());
+                                            break;
+                                        case NO_MATCHED_MSG:
+                                            consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset());
+                                            break;
+                                        default:
+                                    }
+                                } catch (RemotingException e) {
+                                    e.printStackTrace();
+                                } catch (MQBrokerException e) {
+                                    e.printStackTrace();
+                                } catch (Exception e) {
+                                    e.printStackTrace();
+                                }
+                            }
+                        } catch (MQClientException e) {
+                            //reblance error
+                            e.printStackTrace();
+                        } catch (InterruptedException e) {
+                            e.printStackTrace();
+                        } catch (Exception e) {
+                            e.printStackTrace();
+                        }
+                    }
+                }
+
+                public long consumeFromOffset(MessageQueue messageQueue) throws MQClientException {
+                    //-1 when started
+                    long offset = consumer.getOffsetStore().readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY);
+                    if (offset < 0) {
+                        //query from broker
+                        offset = consumer.getOffsetStore().readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE);
+                    }
+                    if (offset < 0) {
+                        //first time start from last offset
+                        offset = consumer.maxOffset(messageQueue);
+                    }
+                    //make sure
+                    if (offset < 0) {
+                        offset = 0;
+                    }
+                    return offset;
+                }
+
+                public void incPullTPS(String topic, int pullSize) {
+                    consumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory()
+                            .getConsumerStatsManager().incPullTPS(consumer.getConsumerGroup(), topic, pullSize);
+                }
+            });
+
+        }
+//        executors.shutdown();
+//        consumer.shutdown();
+    }
 }

filter/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

filter/src/main/java/org/apache/rocketmq/filter/expression/UnaryExpression.java

@@ -53,6 +53,7 @@ public abstract class UnaryExpression implements Expression {
 
     public static Expression createNegate(Expression left) {
         return new UnaryExpression(left, UnaryType.NEGATE) {
+            @Override
             public Object evaluate(EvaluationContext context) throws Exception {
                 Object rvalue = right.evaluate(context);
                 if (rvalue == null) {
@@ -64,6 +65,7 @@ public abstract class UnaryExpression implements Expression {
                 return null;
             }
 
+            @Override
             public String getExpressionSymbol() {
                 return "-";
             }
@@ -85,6 +87,7 @@ public abstract class UnaryExpression implements Expression {
         final Collection inList = t;
 
         return new UnaryInExpression(right, UnaryType.IN, inList, not) {
+            @Override
             public Object evaluate(EvaluationContext context) throws Exception {
 
                 Object rvalue = right.evaluate(context);
@@ -103,8 +106,9 @@ public abstract class UnaryExpression implements Expression {
 
             }
 
+            @Override
             public String toString() {
-                StringBuffer answer = new StringBuffer();
+                StringBuilder answer = new StringBuilder();
                 answer.append(right);
                 answer.append(" ");
                 answer.append(getExpressionSymbol());
@@ -124,6 +128,7 @@ public abstract class UnaryExpression implements Expression {
                 return answer.toString();
             }
 
+            @Override
             public String getExpressionSymbol() {
                 if (not) {
                     return "NOT IN";
@@ -139,6 +144,7 @@ public abstract class UnaryExpression implements Expression {
             super(left, unaryType);
         }
 
+        @Override
         public boolean matches(EvaluationContext context) throws Exception {
             Object object = evaluate(context);
             return object != null && object == Boolean.TRUE;
@@ -147,6 +153,7 @@ public abstract class UnaryExpression implements Expression {
 
     public static BooleanExpression createNOT(BooleanExpression left) {
         return new BooleanUnaryExpression(left, UnaryType.NOT) {
+            @Override
             public Object evaluate(EvaluationContext context) throws Exception {
                 Boolean lvalue = (Boolean) right.evaluate(context);
                 if (lvalue == null) {
@@ -155,6 +162,7 @@ public abstract class UnaryExpression implements Expression {
                 return lvalue.booleanValue() ? Boolean.FALSE : Boolean.TRUE;
             }
 
+            @Override
             public String getExpressionSymbol() {
                 return "NOT";
             }
@@ -163,6 +171,7 @@ public abstract class UnaryExpression implements Expression {
 
     public static BooleanExpression createBooleanCast(Expression left) {
         return new BooleanUnaryExpression(left, UnaryType.BOOLEANCAST) {
+            @Override
             public Object evaluate(EvaluationContext context) throws Exception {
                 Object rvalue = right.evaluate(context);
                 if (rvalue == null) {
@@ -174,10 +183,12 @@ public abstract class UnaryExpression implements Expression {
                 return ((Boolean) rvalue).booleanValue() ? Boolean.TRUE : Boolean.FALSE;
             }
 
+            @Override
             public String toString() {
                 return right.toString();
             }
 
+            @Override
             public String getExpressionSymbol() {
                 return "";
             }
@@ -233,6 +244,7 @@ public abstract class UnaryExpression implements Expression {
     /**
      * @see Object#toString()
      */
+    @Override
     public String toString() {
         return "(" + getExpressionSymbol() + " " + right.toString() + ")";
     }
@@ -240,6 +252,7 @@ public abstract class UnaryExpression implements Expression {
     /**
      * @see Object#hashCode()
      */
+    @Override
     public int hashCode() {
         return toString().hashCode();
     }
@@ -247,6 +260,7 @@ public abstract class UnaryExpression implements Expression {
     /**
      * @see Object#equals(Object)
      */
+    @Override
     public boolean equals(Object o) {
 
         if (o == null || !this.getClass().equals(o.getClass())) {

filter/src/main/java/org/apache/rocketmq/filter/parser/ParseException.java

@@ -106,7 +106,7 @@ public class ParseException extends Exception {
         int[][] expectedTokenSequences,
         String[] tokenImage) {
         String eol = System.getProperty("line.separator", "\n");
-        StringBuffer expected = new StringBuffer();
+        StringBuilder expected = new StringBuilder();
         int maxSize = 0;
         for (int i = 0; i < expectedTokenSequences.length; i++) {
             if (maxSize < expectedTokenSequences[i].length) {
@@ -123,8 +123,9 @@ public class ParseException extends Exception {
         String retval = "Encountered \"";
         Token tok = currentToken.next;
         for (int i = 0; i < maxSize; i++) {
-            if (i != 0)
+            if (i != 0) {
                 retval += " ";
+            }
             if (tok.kind == 0) {
                 retval += tokenImage[0];
                 break;
@@ -157,7 +158,7 @@ public class ParseException extends Exception {
      * string literal.
      */
     static String add_escapes(String str) {
-        StringBuffer retval = new StringBuffer();
+        StringBuilder retval = new StringBuilder();
         char ch;
         for (int i = 0; i < str.length(); i++) {
             switch (str.charAt(i)) {

filter/src/main/java/org/apache/rocketmq/filter/parser/TokenMgrError.java

@@ -66,7 +66,7 @@ public class TokenMgrError extends Error {
      * equivalents in the given string
      */
     protected static final String addEscapes(String str) {
-        StringBuffer retval = new StringBuffer();
+        StringBuilder retval = new StringBuilder();
         char ch;
         for (int i = 0; i < str.length(); i++) {
             switch (str.charAt(i)) {
@@ -141,6 +141,7 @@ public class TokenMgrError extends Error {
      * <p/>
      * from this method for such cases in the release version of your parser.
      */
+    @Override
     public String getMessage() {
         return super.getMessage();
     }

filter/src/test/java/org/apache/rocketmq/filter/ParserTest.java

@@ -84,7 +84,7 @@ public class ParserTest {
     @Test
     public void testParse_floatOverFlow() {
         try {
-            StringBuffer sb = new StringBuffer(210000);
+            StringBuilder sb = new StringBuilder(210000);
             sb.append("1");
             for (int i = 0; i < 2048; i ++) {
                 sb.append("111111111111111111111111111111111111111111111111111");

logappender/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>rocketmq-logappender</artifactId>

logging/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

namesrv/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java

@@ -27,6 +27,8 @@ import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.help.FAQUrl;
+import org.apache.rocketmq.common.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader;
+import org.apache.rocketmq.common.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.common.namesrv.NamesrvUtil;
@@ -103,6 +105,8 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen
                 return this.getBrokerClusterInfo(ctx, request);
             case RequestCode.WIPE_WRITE_PERM_OF_BROKER:
                 return this.wipeWritePermOfBroker(ctx, request);
+            case RequestCode.ADD_WRITE_PERM_OF_BROKER:
+                return this.addWritePermOfBroker(ctx, request);
             case RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER:
                 return getAllTopicListFromNameserver(ctx, request);
             case RequestCode.DELETE_TOPIC_IN_NAMESRV:
@@ -402,6 +406,24 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen
         return response;
     }
 
+    private RemotingCommand addWritePermOfBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(AddWritePermOfBrokerResponseHeader.class);
+        final AddWritePermOfBrokerResponseHeader responseHeader = (AddWritePermOfBrokerResponseHeader) response.readCustomHeader();
+        final AddWritePermOfBrokerRequestHeader requestHeader = (AddWritePermOfBrokerRequestHeader) request.decodeCommandCustomHeader(AddWritePermOfBrokerRequestHeader.class);
+
+        int addTopicCnt = this.namesrvController.getRouteInfoManager().addWritePermOfBrokerByLock(requestHeader.getBrokerName());
+
+        log.info("add write perm of broker[{}], client: {}, {}",
+                requestHeader.getBrokerName(),
+                RemotingHelper.parseChannelRemoteAddr(ctx.channel()),
+                addTopicCnt);
+
+        responseHeader.setAddTopicCount(addTopicCnt);
+        response.setCode(ResponseCode.SUCCESS);
+        response.setRemark(null);
+        return response;
+    }
+
     private RemotingCommand getAllTopicListFromNameserver(ChannelHandlerContext ctx, RemotingCommand request) {
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
 

namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java

@@ -33,6 +33,7 @@ import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.protocol.RequestCode;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.common.namesrv.RegisterBrokerResult;
@@ -252,40 +253,52 @@ public class RouteInfoManager {
     }
 
     public int wipeWritePermOfBrokerByLock(final String brokerName) {
+        return operateWritePermOfBrokerByLock(brokerName, RequestCode.WIPE_WRITE_PERM_OF_BROKER);
+    }
+
+    public int addWritePermOfBrokerByLock(final String brokerName) {
+        return operateWritePermOfBrokerByLock(brokerName, RequestCode.ADD_WRITE_PERM_OF_BROKER);
+    }
+
+    private int operateWritePermOfBrokerByLock(final String brokerName, final int requestCode) {
         try {
             try {
                 this.lock.writeLock().lockInterruptibly();
-                return wipeWritePermOfBroker(brokerName);
+                return operateWritePermOfBroker(brokerName, requestCode);
             } finally {
                 this.lock.writeLock().unlock();
             }
         } catch (Exception e) {
-            log.error("wipeWritePermOfBrokerByLock Exception", e);
+            log.error("operateWritePermOfBrokerByLock Exception", e);
         }
 
         return 0;
     }
 
-    private int wipeWritePermOfBroker(final String brokerName) {
-        int wipeTopicCnt = 0;
-        Iterator<Entry<String, List<QueueData>>> itTopic = this.topicQueueTable.entrySet().iterator();
-        while (itTopic.hasNext()) {
-            Entry<String, List<QueueData>> entry = itTopic.next();
+
+    private int operateWritePermOfBroker(final String brokerName, final int requestCode) {
+        int topicCnt = 0;
+        for (Entry<String, List<QueueData>> entry : this.topicQueueTable.entrySet()) {
             List<QueueData> qdList = entry.getValue();
 
-            Iterator<QueueData> it = qdList.iterator();
-            while (it.hasNext()) {
-                QueueData qd = it.next();
+            for (QueueData qd : qdList) {
                 if (qd.getBrokerName().equals(brokerName)) {
                     int perm = qd.getPerm();
-                    perm &= ~PermName.PERM_WRITE;
+                    switch (requestCode) {
+                        case RequestCode.WIPE_WRITE_PERM_OF_BROKER:
+                            perm &= ~PermName.PERM_WRITE;
+                            break;
+                        case RequestCode.ADD_WRITE_PERM_OF_BROKER:
+                            perm = PermName.PERM_READ | PermName.PERM_WRITE;
+                            break;
+                    }
                     qd.setPerm(perm);
-                    wipeTopicCnt++;
+                    topicCnt++;
                 }
             }
         }
 
-        return wipeTopicCnt;
+        return topicCnt;
     }
 
     public void unregisterBroker(

namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerTest.java

@@ -17,17 +17,23 @@
 package org.apache.rocketmq.namesrv.routeinfo;
 
 import io.netty.channel.Channel;
-import java.util.ArrayList;
-import java.util.concurrent.ConcurrentHashMap;
 import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.PermName;
 import org.apache.rocketmq.common.namesrv.RegisterBrokerResult;
 import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
+import org.apache.rocketmq.common.protocol.route.QueueData;
 import org.apache.rocketmq.common.protocol.route.TopicRouteData;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 
@@ -74,14 +80,28 @@ public class RouteInfoManagerTest {
         topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap);
         Channel channel = mock(Channel.class);
         RegisterBrokerResult registerBrokerResult = routeInfoManager.registerBroker("default-cluster", "127.0.0.1:10911", "default-broker", 1234, "127.0.0.1:1001",
-            topicConfigSerializeWrapper, new ArrayList<String>(), channel);
+                topicConfigSerializeWrapper, new ArrayList<String>(), channel);
         assertThat(registerBrokerResult).isNotNull();
     }
 
     @Test
-    public void testWipeWritePermOfBrokerByLock() {
-        int result = routeInfoManager.wipeWritePermOfBrokerByLock("default-broker");
-        assertThat(result).isEqualTo(0);
+    public void testWipeWritePermOfBrokerByLock() throws Exception {
+        List<QueueData> qdList = new ArrayList<>();
+        QueueData qd = new QueueData();
+        qd.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);
+        qd.setBrokerName("broker-a");
+        qdList.add(qd);
+        HashMap<String, List<QueueData>> topicQueueTable = new HashMap<>();
+        topicQueueTable.put("topic-a", qdList);
+
+        Field filed = RouteInfoManager.class.getDeclaredField("topicQueueTable");
+        filed.setAccessible(true);
+        filed.set(routeInfoManager, topicQueueTable);
+
+        int addTopicCnt = routeInfoManager.wipeWritePermOfBrokerByLock("broker-a");
+        assertThat(addTopicCnt).isEqualTo(1);
+        assertThat(qd.getPerm()).isEqualTo(PermName.PERM_READ);
+
     }
 
     @Test
@@ -119,4 +139,24 @@ public class RouteInfoManagerTest {
         byte[] topicList = routeInfoManager.getHasUnitSubUnUnitTopicList();
         assertThat(topicList).isNotNull();
     }
+
+    @Test
+    public void testAddWritePermOfBrokerByLock() throws Exception {
+        List<QueueData> qdList = new ArrayList<>();
+        QueueData qd = new QueueData();
+        qd.setPerm(PermName.PERM_READ);
+        qd.setBrokerName("broker-a");
+        qdList.add(qd);
+        HashMap<String, List<QueueData>> topicQueueTable = new HashMap<>();
+        topicQueueTable.put("topic-a", qdList);
+
+        Field filed = RouteInfoManager.class.getDeclaredField("topicQueueTable");
+        filed.setAccessible(true);
+        filed.set(routeInfoManager, topicQueueTable);
+
+        int addTopicCnt = routeInfoManager.addWritePermOfBrokerByLock("broker-a");
+        assertThat(addTopicCnt).isEqualTo(1);
+        assertThat(qd.getPerm()).isEqualTo(PermName.PERM_READ | PermName.PERM_WRITE);
+
+    }
 }
\ No newline at end of file

openmessaging/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

pom.xml

@@ -29,7 +29,7 @@
     <inceptionYear>2012</inceptionYear>
     <groupId>org.apache.rocketmq</groupId>
     <artifactId>rocketmq-all</artifactId>
-    <version>4.9.2-SNAPSHOT</version>
+    <version>4.9.3-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>Apache RocketMQ ${project.version}</name>
     <url>http://rocketmq.apache.org/</url>

remoting/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java

@@ -36,7 +36,7 @@ public class RemotingHelper {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(ROCKETMQ_REMOTING);
 
     public static String exceptionSimpleDesc(final Throwable e) {
-        StringBuffer sb = new StringBuffer();
+        StringBuilder sb = new StringBuilder();
         if (e != null) {
             sb.append(e.toString());
 

remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java

@@ -165,6 +165,10 @@ public class RemotingUtil {
         return sb.toString();
     }
 
+    public static String convert2IpString(final String addr) {
+        return socketAddress2String(string2SocketAddress(addr));
+    }
+
     private static boolean isBridge(NetworkInterface networkInterface) {
         try {
             if (isLinuxPlatform()) {

remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java

@@ -199,11 +199,12 @@ public abstract class NettyRemotingAbstract {
                 @Override
                 public void run() {
                     try {
-                        doBeforeRpcHooks(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd);
+                        String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel());
+                        doBeforeRpcHooks(remoteAddr, cmd);
                         final RemotingResponseCallback callback = new RemotingResponseCallback() {
                             @Override
                             public void callback(RemotingCommand response) {
-                                doAfterRpcHooks(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd, response);
+                                doAfterRpcHooks(remoteAddr, cmd, response);
                                 if (!cmd.isOnewayRPC()) {
                                     if (response != null) {
                                         response.setOpaque(opaque);

srvutil/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

store/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

store/src/main/java/org/apache/rocketmq/store/CommitLog.java

@@ -22,10 +22,12 @@ import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.nio.ByteBuffer;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Supplier;
 
@@ -43,6 +45,7 @@ import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.config.FlushDiskType;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.ha.HAService;
 import org.apache.rocketmq.store.schedule.ScheduleMessageService;
 
@@ -71,9 +74,20 @@ public class CommitLog {
 
     protected final PutMessageLock putMessageLock;
 
+    private volatile Set<String> fullStorePaths = Collections.emptySet();
+
     public CommitLog(final DefaultMessageStore defaultMessageStore) {
-        this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(),
-            defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(), defaultMessageStore.getAllocateMappedFileService());
+        String storePath = defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog();
+        if (storePath.contains(MessageStoreConfig.MULTI_PATH_SPLITTER)) {
+            this.mappedFileQueue = new MultiPathMappedFileQueue(defaultMessageStore.getMessageStoreConfig(),
+                    defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),
+                    defaultMessageStore.getAllocateMappedFileService(), this::getFullStorePaths);
+        } else {
+            this.mappedFileQueue = new MappedFileQueue(storePath,
+                    defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(),
+                    defaultMessageStore.getAllocateMappedFileService());
+        }
+
         this.defaultMessageStore = defaultMessageStore;
 
         if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
@@ -95,6 +109,14 @@ public class CommitLog {
 
     }
 
+    public void setFullStorePaths(Set<String> fullStorePaths) {
+        this.fullStorePaths = fullStorePaths;
+    }
+
+    public Set<String> getFullStorePaths() {
+        return fullStorePaths;
+    }
+
     public boolean load() {
         boolean result = this.mappedFileQueue.load();
         log.info("load commit log " + (result ? "OK" : "Failed"));
@@ -685,8 +707,8 @@ public class CommitLog {
         PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result);
 
         // Statistics
-        storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).incrementAndGet();
-        storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes());
+        storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).add(1);
+        storeStatsService.getSinglePutMessageTopicSizeTotal(topic).add(result.getWroteBytes());
 
         CompletableFuture<PutMessageStatus> flushResultFuture = submitFlushRequest(result, msg);
         CompletableFuture<PutMessageStatus> replicaResultFuture = submitReplicaRequest(result, msg);
@@ -802,8 +824,8 @@ public class CommitLog {
         PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result);
 
         // Statistics
-        storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).addAndGet(result.getMsgNum());
-        storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).addAndGet(result.getWroteBytes());
+        storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).add(result.getMsgNum());
+        storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).add(result.getWroteBytes());
 
         CompletableFuture<PutMessageStatus> flushOKFuture = submitFlushRequest(result, messageExtBatch);
         CompletableFuture<PutMessageStatus> replicaOKFuture = submitReplicaRequest(result, messageExtBatch);

store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java

@@ -26,6 +26,7 @@ import java.nio.ByteBuffer;
 import java.nio.channels.FileLock;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.Map;
@@ -163,6 +164,8 @@ public class DefaultMessageStore implements MessageStore {
 
         File file = new File(StorePathConfigHelper.getLockFile(messageStoreConfig.getStorePathRootDir()));
         MappedFile.ensureDirOK(file.getParent());
+        MappedFile.ensureDirOK(getStorePathPhysic());
+        MappedFile.ensureDirOK(getStorePathLogic());
         lockFile = new RandomAccessFile(file, "rw");
     }
 
@@ -186,10 +189,6 @@ public class DefaultMessageStore implements MessageStore {
             boolean lastExitOK = !this.isTempFileExist();
             log.info("last shutdown {}", lastExitOK ? "normally" : "abnormally");
 
-            if (null != scheduleMessageService) {
-                result = result && this.scheduleMessageService.load();
-            }
-
             // load Commit Log
             result = result && this.commitLog.load();
 
@@ -205,7 +204,12 @@ public class DefaultMessageStore implements MessageStore {
                 this.recover(lastExitOK);
 
                 log.info("load over, and the max phy offset = {}", this.getMaxPhyOffset());
+
+                if (null != scheduleMessageService) {
+                    result =  this.scheduleMessageService.load();
+                }
             }
+
         } catch (Exception e) {
             log.error("load exception", e);
             result = false;
@@ -438,7 +442,7 @@ public class DefaultMessageStore implements MessageStore {
             this.storeStatsService.setPutMessageEntireTimeMax(elapsedTime);
 
             if (null == result || !result.isOk()) {
-                this.storeStatsService.getPutMessageFailedTimes().incrementAndGet();
+                this.storeStatsService.getPutMessageFailedTimes().add(1);
             }
         });
 
@@ -468,7 +472,7 @@ public class DefaultMessageStore implements MessageStore {
             this.storeStatsService.setPutMessageEntireTimeMax(elapsedTime);
 
             if (null == result || !result.isOk()) {
-                this.storeStatsService.getPutMessageFailedTimes().incrementAndGet();
+                this.storeStatsService.getPutMessageFailedTimes().add(1);
             }
         });
 
@@ -535,7 +539,8 @@ public class DefaultMessageStore implements MessageStore {
         long minOffset = 0;
         long maxOffset = 0;
 
-        GetMessageResult getResult = new GetMessageResult();
+        // lazy init when find msg.
+        GetMessageResult getResult = null;
 
         final long maxOffsetPy = this.commitLog.getMaxOffset();
 
@@ -572,6 +577,9 @@ public class DefaultMessageStore implements MessageStore {
                         int i = 0;
                         final int maxFilterMessageCount = Math.max(16000, maxMsgNums * ConsumeQueue.CQ_STORE_UNIT_SIZE);
                         final boolean diskFallRecorded = this.messageStoreConfig.isDiskFallRecorded();
+
+                        getResult = new GetMessageResult(maxMsgNums);
+
                         ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();
                         for (; i < bufferConsumeQueue.getSize() && i < maxFilterMessageCount; i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
                             long offsetPy = bufferConsumeQueue.getByteBuffer().getLong();
@@ -634,7 +642,7 @@ public class DefaultMessageStore implements MessageStore {
                                 continue;
                             }
 
-                            this.storeStatsService.getGetMessageTransferedMsgCount().incrementAndGet();
+                            this.storeStatsService.getGetMessageTransferedMsgCount().add(1);
                             getResult.addMessage(selectResult);
                             status = GetMessageStatus.FOUND;
                             nextPhyFileStartOffset = Long.MIN_VALUE;
@@ -668,13 +676,18 @@ public class DefaultMessageStore implements MessageStore {
         }
 
         if (GetMessageStatus.FOUND == status) {
-            this.storeStatsService.getGetMessageTimesTotalFound().incrementAndGet();
+            this.storeStatsService.getGetMessageTimesTotalFound().add(1);
         } else {
-            this.storeStatsService.getGetMessageTimesTotalMiss().incrementAndGet();
+            this.storeStatsService.getGetMessageTimesTotalMiss().add(1);
         }
         long elapsedTime = this.getSystemClock().now() - beginTime;
         this.storeStatsService.setGetMessageEntireTimeMax(elapsedTime);
 
+        // lazy init no data found.
+        if (getResult == null) {
+            getResult = new GetMessageResult(0);
+        }
+
         getResult.setStatus(status);
         getResult.setNextBeginOffset(nextBeginOffset);
         getResult.setMaxOffset(maxOffset);
@@ -768,8 +781,8 @@ public class DefaultMessageStore implements MessageStore {
         return this.storeStatsService.toString();
     }
 
-    private String getStorePathPhysic() {
-        String storePathPhysic = "";
+    public String getStorePathPhysic() {
+        String storePathPhysic;
         if (DefaultMessageStore.this.getMessageStoreConfig().isEnableDLegerCommitLog()) {
             storePathPhysic = ((DLedgerCommitLog)DefaultMessageStore.this.getCommitLog()).getdLedgerServer().getdLedgerConfig().getDataStorePath();
         } else {
@@ -778,20 +791,29 @@ public class DefaultMessageStore implements MessageStore {
         return storePathPhysic;
     }
 
+    public String getStorePathLogic() {
+        return StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir());
+    }
+
     @Override
     public HashMap<String, String> getRuntimeInfo() {
         HashMap<String, String> result = this.storeStatsService.getRuntimeInfo();
 
         {
-            double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(getStorePathPhysic());
-            result.put(RunningStats.commitLogDiskRatio.name(), String.valueOf(physicRatio));
-
+            double minPhysicsUsedRatio = Double.MAX_VALUE;
+            String commitLogStorePath = getStorePathPhysic();
+            String[] paths = commitLogStorePath.trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+            for (String clPath : paths) {
+                double physicRatio = UtilAll.isPathExists(clPath) ?
+                        UtilAll.getDiskPartitionSpaceUsedPercent(clPath) : -1;
+                result.put(RunningStats.commitLogDiskRatio.name() + "_" + clPath, String.valueOf(physicRatio));
+                minPhysicsUsedRatio = Math.min(minPhysicsUsedRatio, physicRatio);
+            }
+            result.put(RunningStats.commitLogDiskRatio.name(), String.valueOf(minPhysicsUsedRatio));
         }
 
         {
-
-            String storePathLogics = StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir());
-            double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics);
+            double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(getStorePathLogic());
             result.put(RunningStats.consumeQueueDiskRatio.name(), String.valueOf(logicsRatio));
         }
 
@@ -1650,25 +1672,43 @@ public class DefaultMessageStore implements MessageStore {
             cleanImmediately = false;
 
             {
-                double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(getStorePathPhysic());
-                if (physicRatio > diskSpaceWarningLevelRatio) {
+                String commitLogStorePath = DefaultMessageStore.this.getMessageStoreConfig().getStorePathCommitLog();
+                String[] storePaths = commitLogStorePath.trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+                Set<String> fullStorePath = new HashSet<>();
+                double minPhysicRatio = 100;
+                String minStorePath = null;
+                for (String storePathPhysic : storePaths) {
+                    double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic);
+                    if (minPhysicRatio > physicRatio) {
+                        minPhysicRatio =  physicRatio;
+                        minStorePath = storePathPhysic;
+                    }
+                    if (physicRatio > diskSpaceCleanForciblyRatio) {
+                        fullStorePath.add(storePathPhysic);
+                    }
+                }
+                DefaultMessageStore.this.commitLog.setFullStorePaths(fullStorePath);
+                if (minPhysicRatio > diskSpaceWarningLevelRatio) {
                     boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskFull();
                     if (diskok) {
-                        DefaultMessageStore.log.error("physic disk maybe full soon " + physicRatio + ", so mark disk full");
+                        DefaultMessageStore.log.error("physic disk maybe full soon " + minPhysicRatio +
+                                ", so mark disk full, storePathPhysic=" + minStorePath);
                     }
 
                     cleanImmediately = true;
-                } else if (physicRatio > diskSpaceCleanForciblyRatio) {
+                } else if (minPhysicRatio > diskSpaceCleanForciblyRatio) {
                     cleanImmediately = true;
                 } else {
                     boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskOK();
                     if (!diskok) {
-                        DefaultMessageStore.log.info("physic disk space OK " + physicRatio + ", so mark disk ok");
+                        DefaultMessageStore.log.info("physic disk space OK " + minPhysicRatio +
+                                ", so mark disk ok, storePathPhysic=" + minStorePath);
                     }
                 }
 
-                if (physicRatio < 0 || physicRatio > ratio) {
-                    DefaultMessageStore.log.info("physic disk maybe full soon, so reclaim space, " + physicRatio);
+                if (minPhysicRatio < 0 || minPhysicRatio > ratio) {
+                    DefaultMessageStore.log.info("physic disk maybe full soon, so reclaim space, "
+                            + minPhysicRatio + ", storePathPhysic=" + minStorePath);
                     return true;
                 }
             }
@@ -1709,8 +1749,27 @@ public class DefaultMessageStore implements MessageStore {
         public void setManualDeleteFileSeveralTimes(int manualDeleteFileSeveralTimes) {
             this.manualDeleteFileSeveralTimes = manualDeleteFileSeveralTimes;
         }
+
+        public double calcStorePathPhysicRatio() {
+            Set<String> fullStorePath = new HashSet<>();
+            String storePath = getStorePathPhysic();
+            String[] paths = storePath.trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+            double minPhysicRatio = 100;
+            for (String path : paths) {
+                double physicRatio = UtilAll.isPathExists(path) ?
+                        UtilAll.getDiskPartitionSpaceUsedPercent(path) : -1;
+                minPhysicRatio = Math.min(minPhysicRatio, physicRatio);
+                if (physicRatio > diskSpaceCleanForciblyRatio) {
+                    fullStorePath.add(path);
+                }
+            }
+            DefaultMessageStore.this.commitLog.setFullStorePaths(fullStorePath);
+            return minPhysicRatio;
+
+        }
+
         public boolean isSpaceFull() {
-            double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(getStorePathPhysic());
+            double physicRatio = calcStorePathPhysicRatio();
             double ratio = DefaultMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0;
             if (physicRatio > ratio) {
                 DefaultMessageStore.log.info("physic disk of commitLog used: " + physicRatio);
@@ -1922,10 +1981,10 @@ public class DefaultMessageStore implements MessageStore {
                                     readSize += size;
                                     if (DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {
                                         DefaultMessageStore.this.storeStatsService
-                                            .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).incrementAndGet();
+                                            .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1);
                                         DefaultMessageStore.this.storeStatsService
                                             .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic())
-                                            .addAndGet(dispatchRequest.getMsgSize());
+                                            .add(dispatchRequest.getMsgSize());
                                     }
                                 } else if (size == 0) {
                                     this.reputFromOffset = DefaultMessageStore.this.commitLog.rollNextFile(this.reputFromOffset);

store/src/main/java/org/apache/rocketmq/store/GetMessageResult.java

@@ -23,10 +23,8 @@ import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
 public class GetMessageResult {
 
-    private final List<SelectMappedBufferResult> messageMapedList =
-        new ArrayList<SelectMappedBufferResult>(100);
-
-    private final List<ByteBuffer> messageBufferList = new ArrayList<ByteBuffer>(100);
+    private final List<SelectMappedBufferResult> messageMapedList;
+    private final List<ByteBuffer> messageBufferList;
 
     private GetMessageStatus status;
     private long nextBeginOffset;
@@ -40,6 +38,13 @@ public class GetMessageResult {
     private int msgCount4Commercial = 0;
 
     public GetMessageResult() {
+        messageMapedList = new ArrayList<>(100);
+        messageBufferList = new ArrayList<>(100);
+    }
+
+    public GetMessageResult(int resultSize) {
+        messageMapedList = new ArrayList<>(resultSize);
+        messageBufferList = new ArrayList<>(resultSize);
     }
 
     public GetMessageStatus getStatus() {

store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java

@@ -20,6 +20,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.ListIterator;
@@ -37,13 +38,13 @@ public class MappedFileQueue {
 
     private final String storePath;
 
-    private final int mappedFileSize;
+    protected final int mappedFileSize;
 
-    private final CopyOnWriteArrayList<MappedFile> mappedFiles = new CopyOnWriteArrayList<MappedFile>();
+    protected final CopyOnWriteArrayList<MappedFile> mappedFiles = new CopyOnWriteArrayList<MappedFile>();
 
     private final AllocateMappedFileService allocateMappedFileService;
 
-    private long flushedWhere = 0;
+    protected long flushedWhere = 0;
     private long committedWhere = 0;
 
     private volatile long storeTimestamp = 0;
@@ -144,35 +145,40 @@ public class MappedFileQueue {
         }
     }
 
+
     public boolean load() {
         File dir = new File(this.storePath);
-        File[] files = dir.listFiles();
-        if (files != null) {
-            // ascending order
-            Arrays.sort(files);
-            for (File file : files) {
-
-                if (file.length() != this.mappedFileSize) {
-                    log.warn(file + "\t" + file.length()
-                        + " length not matched message store config value, please check it manually");
-                    return false;
-                }
+        File[] ls = dir.listFiles();
+        if (ls != null) {
+            return doLoad(Arrays.asList(ls));
+        }
+        return true;
+    }
 
-                try {
-                    MappedFile mappedFile = new MappedFile(file.getPath(), mappedFileSize);
-
-                    mappedFile.setWrotePosition(this.mappedFileSize);
-                    mappedFile.setFlushedPosition(this.mappedFileSize);
-                    mappedFile.setCommittedPosition(this.mappedFileSize);
-                    this.mappedFiles.add(mappedFile);
-                    log.info("load " + file.getPath() + " OK");
-                } catch (IOException e) {
-                    log.error("load file " + file + " error", e);
-                    return false;
-                }
+    public boolean doLoad(List<File> files) {
+        // ascending order
+        files.sort(Comparator.comparing(File::getName));
+
+        for (File file : files) {
+            if (file.length() != this.mappedFileSize) {
+                log.warn(file + "\t" + file.length()
+                        + " length not matched message store config value, ignore it");
+                return true;
             }
-        }
 
+            try {
+                MappedFile mappedFile = new MappedFile(file.getPath(), mappedFileSize);
+
+                mappedFile.setWrotePosition(this.mappedFileSize);
+                mappedFile.setFlushedPosition(this.mappedFileSize);
+                mappedFile.setCommittedPosition(this.mappedFileSize);
+                this.mappedFiles.add(mappedFile);
+                log.info("load " + file.getPath() + " OK");
+            } catch (IOException e) {
+                log.error("load file " + file + " error", e);
+                return false;
+            }
+        }
         return true;
     }
 
@@ -204,33 +210,41 @@ public class MappedFileQueue {
         }
 
         if (createOffset != -1 && needCreate) {
-            String nextFilePath = this.storePath + File.separator + UtilAll.offset2FileName(createOffset);
-            String nextNextFilePath = this.storePath + File.separator
-                + UtilAll.offset2FileName(createOffset + this.mappedFileSize);
-            MappedFile mappedFile = null;
+            return tryCreateMappedFile(createOffset);
+        }
 
-            if (this.allocateMappedFileService != null) {
-                mappedFile = this.allocateMappedFileService.putRequestAndReturnMappedFile(nextFilePath,
+        return mappedFileLast;
+    }
+
+    protected MappedFile tryCreateMappedFile(long createOffset) {
+        String nextFilePath = this.storePath + File.separator + UtilAll.offset2FileName(createOffset);
+        String nextNextFilePath = this.storePath + File.separator + UtilAll.offset2FileName(createOffset
+                + this.mappedFileSize);
+        return doCreateMappedFile(nextFilePath, nextNextFilePath);
+    }
+
+    protected MappedFile doCreateMappedFile(String nextFilePath, String nextNextFilePath) {
+        MappedFile mappedFile = null;
+
+        if (this.allocateMappedFileService != null) {
+            mappedFile = this.allocateMappedFileService.putRequestAndReturnMappedFile(nextFilePath,
                     nextNextFilePath, this.mappedFileSize);
-            } else {
-                try {
-                    mappedFile = new MappedFile(nextFilePath, this.mappedFileSize);
-                } catch (IOException e) {
-                    log.error("create mappedFile exception", e);
-                }
+        } else {
+            try {
+                mappedFile = new MappedFile(nextFilePath, this.mappedFileSize);
+            } catch (IOException e) {
+                log.error("create mappedFile exception", e);
             }
+        }
 
-            if (mappedFile != null) {
-                if (this.mappedFiles.isEmpty()) {
-                    mappedFile.setFirstCreateInQueue(true);
-                }
-                this.mappedFiles.add(mappedFile);
+        if (mappedFile != null) {
+            if (this.mappedFiles.isEmpty()) {
+                mappedFile.setFirstCreateInQueue(true);
             }
-
-            return mappedFile;
+            this.mappedFiles.add(mappedFile);
         }
 
-        return mappedFileLast;
+        return mappedFile;
     }
 
     public MappedFile getLastMappedFile(final long startOffset) {

store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store;
+
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class MultiPathMappedFileQueue extends MappedFileQueue {
+
+    private final MessageStoreConfig config;
+    private final Supplier<Set<String>> fullStorePathsSupplier;
+
+    public MultiPathMappedFileQueue(MessageStoreConfig messageStoreConfig, int mappedFileSize,
+                                    AllocateMappedFileService allocateMappedFileService,
+                                    Supplier<Set<String>> fullStorePathsSupplier) {
+        super(messageStoreConfig.getStorePathCommitLog(), mappedFileSize, allocateMappedFileService);
+        this.config = messageStoreConfig;
+        this.fullStorePathsSupplier = fullStorePathsSupplier;
+    }
+
+    private Set<String> getPaths() {
+        String[] paths = config.getStorePathCommitLog().trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+        return new HashSet<>(Arrays.asList(paths));
+    }
+
+    private Set<String> getReadonlyPaths() {
+        String pathStr = config.getReadOnlyCommitLogStorePaths();
+        if (StringUtils.isBlank(pathStr)) {
+            return Collections.emptySet();
+        }
+        String[] paths = pathStr.trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+        return new HashSet<>(Arrays.asList(paths));
+    }
+
+    @Override
+    public boolean load() {
+        Set<String> storePathSet = getPaths();
+        storePathSet.addAll(getReadonlyPaths());
+
+        List<File> files = new ArrayList<>();
+        for (String path : storePathSet) {
+            File dir = new File(path);
+            File[] ls = dir.listFiles();
+            if (ls != null) {
+                Collections.addAll(files, ls);
+            }
+        }
+
+        return doLoad(files);
+    }
+
+    @Override
+    protected MappedFile tryCreateMappedFile(long createOffset) {
+        long fileIdx = createOffset / this.mappedFileSize;
+        Set<String> storePath = getPaths();
+        Set<String> readonlyPathSet = getReadonlyPaths();
+        Set<String> fullStorePaths =
+                fullStorePathsSupplier == null ? Collections.emptySet() : fullStorePathsSupplier.get();
+
+
+        HashSet<String> availableStorePath = new HashSet<>(storePath);
+        //do not create file in readonly store path.
+        availableStorePath.removeAll(readonlyPathSet);
+
+        //do not create file is space is nearly full.
+        availableStorePath.removeAll(fullStorePaths);
+
+        //if no store path left, fall back to writable store path.
+        if (availableStorePath.isEmpty()) {
+            availableStorePath = new HashSet<>(storePath);
+            availableStorePath.removeAll(readonlyPathSet);
+        }
+
+        String[] paths = availableStorePath.toArray(new String[]{});
+        Arrays.sort(paths);
+        String nextFilePath = paths[(int) (fileIdx % paths.length)] + File.separator
+                + UtilAll.offset2FileName(createOffset);
+        String nextNextFilePath = paths[(int) ((fileIdx + 1) % paths.length)] + File.separator
+                + UtilAll.offset2FileName(createOffset + this.mappedFileSize);
+        return doCreateMappedFile(nextFilePath, nextNextFilePath);
+    }
+
+    @Override
+    public void destroy() {
+        for (MappedFile mf : this.mappedFiles) {
+            mf.destroy(1000 * 3);
+        }
+        this.mappedFiles.clear();
+        this.flushedWhere = 0;
+
+
+        Set<String> storePathSet = getPaths();
+        storePathSet.addAll(getReadonlyPaths());
+
+        for (String path : storePathSet) {
+            File file = new File(path);
+            if (file.isDirectory()) {
+                file.delete();
+            }
+        }
+    }
+}

store/src/main/java/org/apache/rocketmq/store/StoreStatsService.java

@@ -22,7 +22,7 @@ import java.util.LinkedList;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
 import java.util.concurrent.locks.ReentrantLock;
 import org.apache.rocketmq.common.ServiceThread;
 import org.apache.rocketmq.common.constant.LoggerName;
@@ -41,23 +41,23 @@ public class StoreStatsService extends ServiceThread {
 
     private static int printTPSInterval = 60 * 1;
 
-    private final AtomicLong putMessageFailedTimes = new AtomicLong(0);
+    private final LongAdder putMessageFailedTimes = new LongAdder();
 
-    private final ConcurrentMap<String, AtomicLong> putMessageTopicTimesTotal =
-        new ConcurrentHashMap<String, AtomicLong>(128);
-    private final ConcurrentMap<String, AtomicLong> putMessageTopicSizeTotal =
-        new ConcurrentHashMap<String, AtomicLong>(128);
+    private final ConcurrentMap<String, LongAdder> putMessageTopicTimesTotal =
+        new ConcurrentHashMap<>(128);
+    private final ConcurrentMap<String, LongAdder> putMessageTopicSizeTotal =
+        new ConcurrentHashMap<>(128);
 
-    private final AtomicLong getMessageTimesTotalFound = new AtomicLong(0);
-    private final AtomicLong getMessageTransferedMsgCount = new AtomicLong(0);
-    private final AtomicLong getMessageTimesTotalMiss = new AtomicLong(0);
+    private final LongAdder getMessageTimesTotalFound = new LongAdder();
+    private final LongAdder getMessageTransferedMsgCount = new LongAdder();
+    private final LongAdder getMessageTimesTotalMiss = new LongAdder();
     private final LinkedList<CallSnapshot> putTimesList = new LinkedList<CallSnapshot>();
 
     private final LinkedList<CallSnapshot> getTimesFoundList = new LinkedList<CallSnapshot>();
     private final LinkedList<CallSnapshot> getTimesMissList = new LinkedList<CallSnapshot>();
     private final LinkedList<CallSnapshot> transferedMsgCountList = new LinkedList<CallSnapshot>();
-    private volatile AtomicLong[] putMessageDistributeTime;
-    private volatile AtomicLong[] lastPutMessageDistributeTime;
+    private volatile LongAdder[] putMessageDistributeTime;
+    private volatile LongAdder[] lastPutMessageDistributeTime;
     private long messageStoreBootTimestamp = System.currentTimeMillis();
     private volatile long putMessageEntireTimeMax = 0;
     private volatile long getMessageEntireTimeMax = 0;
@@ -75,10 +75,10 @@ public class StoreStatsService extends ServiceThread {
         this.initPutMessageDistributeTime();
     }
 
-    private AtomicLong[] initPutMessageDistributeTime() {
-        AtomicLong[] next = new AtomicLong[13];
+    private LongAdder[] initPutMessageDistributeTime() {
+        LongAdder[] next = new LongAdder[13];
         for (int i = 0; i < next.length; i++) {
-            next[i] = new AtomicLong(0);
+            next[i] = new LongAdder();
         }
 
         this.lastPutMessageDistributeTime = this.putMessageDistributeTime;
@@ -93,48 +93,48 @@ public class StoreStatsService extends ServiceThread {
     }
 
     public void setPutMessageEntireTimeMax(long value) {
-        final AtomicLong[] times = this.putMessageDistributeTime;
+        final LongAdder[] times = this.putMessageDistributeTime;
 
         if (null == times)
             return;
 
         // us
         if (value <= 0) {
-            times[0].incrementAndGet();
+            times[0].add(1);
         } else if (value < 10) {
-            times[1].incrementAndGet();
+            times[1].add(1);
         } else if (value < 50) {
-            times[2].incrementAndGet();
+            times[2].add(1);
         } else if (value < 100) {
-            times[3].incrementAndGet();
+            times[3].add(1);
         } else if (value < 200) {
-            times[4].incrementAndGet();
+            times[4].add(1);
         } else if (value < 500) {
-            times[5].incrementAndGet();
+            times[5].add(1);
         } else if (value < 1000) {
-            times[6].incrementAndGet();
+            times[6].add(1);
         }
         // 2s
         else if (value < 2000) {
-            times[7].incrementAndGet();
+            times[7].add(1);
         }
         // 3s
         else if (value < 3000) {
-            times[8].incrementAndGet();
+            times[8].add(1);
         }
         // 4s
         else if (value < 4000) {
-            times[9].incrementAndGet();
+            times[9].add(1);
         }
         // 5s
         else if (value < 5000) {
-            times[10].incrementAndGet();
+            times[10].add(1);
         }
         // 10s
         else if (value < 10000) {
-            times[11].incrementAndGet();
+            times[11].add(1);
         } else {
-            times[12].incrementAndGet();
+            times[12].add(1);
         }
 
         if (value > this.putMessageEntireTimeMax) {
@@ -194,8 +194,8 @@ public class StoreStatsService extends ServiceThread {
 
     public long getPutMessageTimesTotal() {
         long rs = 0;
-        for (AtomicLong data : putMessageTopicTimesTotal.values()) {
-            rs += data.get();
+        for (LongAdder data : putMessageTopicTimesTotal.values()) {
+            rs += data.longValue();
         }
         return rs;
     }
@@ -218,8 +218,8 @@ public class StoreStatsService extends ServiceThread {
 
     public long getPutMessageSizeTotal() {
         long rs = 0;
-        for (AtomicLong data : putMessageTopicSizeTotal.values()) {
-            rs += data.get();
+        for (LongAdder data : putMessageTopicSizeTotal.values()) {
+            rs += data.longValue();
         }
         return rs;
     }
@@ -299,13 +299,13 @@ public class StoreStatsService extends ServiceThread {
     }
 
     private String putMessageDistributeTimeToString() {
-        final AtomicLong[] times = this.lastPutMessageDistributeTime;
+        final LongAdder[] times = this.lastPutMessageDistributeTime;
         if (null == times)
             return null;
 
         final StringBuilder sb = new StringBuilder();
         for (int i = 0; i < times.length; i++) {
-            long value = times[i].get();
+            long value = times[i].longValue();
             sb.append(String.format("%s:%d", PUT_MESSAGE_ENTIRE_TIME_MAX_DESC[i], value));
             sb.append(" ");
         }
@@ -477,19 +477,19 @@ public class StoreStatsService extends ServiceThread {
             }
 
             this.getTimesFoundList.add(new CallSnapshot(System.currentTimeMillis(),
-                this.getMessageTimesTotalFound.get()));
+                this.getMessageTimesTotalFound.longValue()));
             if (this.getTimesFoundList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) {
                 this.getTimesFoundList.removeFirst();
             }
 
             this.getTimesMissList.add(new CallSnapshot(System.currentTimeMillis(),
-                this.getMessageTimesTotalMiss.get()));
+                this.getMessageTimesTotalMiss.longValue()));
             if (this.getTimesMissList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) {
                 this.getTimesMissList.removeFirst();
             }
 
             this.transferedMsgCountList.add(new CallSnapshot(System.currentTimeMillis(),
-                this.getMessageTransferedMsgCount.get()));
+                this.getMessageTransferedMsgCount.longValue()));
             if (this.transferedMsgCountList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) {
                 this.transferedMsgCountList.removeFirst();
             }
@@ -510,14 +510,14 @@ public class StoreStatsService extends ServiceThread {
                 this.getGetTransferedTps(printTPSInterval)
             );
 
-            final AtomicLong[] times = this.initPutMessageDistributeTime();
+            final LongAdder[] times = this.initPutMessageDistributeTime();
             if (null == times)
                 return;
 
             final StringBuilder sb = new StringBuilder();
             long totalPut = 0;
             for (int i = 0; i < times.length; i++) {
-                long value = times[i].get();
+                long value = times[i].longValue();
                 totalPut += value;
                 sb.append(String.format("%s:%d", PUT_MESSAGE_ENTIRE_TIME_MAX_DESC[i], value));
                 sb.append(" ");
@@ -527,27 +527,27 @@ public class StoreStatsService extends ServiceThread {
         }
     }
 
-    public AtomicLong getGetMessageTimesTotalFound() {
+    public LongAdder getGetMessageTimesTotalFound() {
         return getMessageTimesTotalFound;
     }
 
-    public AtomicLong getGetMessageTimesTotalMiss() {
+    public LongAdder getGetMessageTimesTotalMiss() {
         return getMessageTimesTotalMiss;
     }
 
-    public AtomicLong getGetMessageTransferedMsgCount() {
+    public LongAdder getGetMessageTransferedMsgCount() {
         return getMessageTransferedMsgCount;
     }
 
-    public AtomicLong getPutMessageFailedTimes() {
+    public LongAdder getPutMessageFailedTimes() {
         return putMessageFailedTimes;
     }
 
-    public AtomicLong getSinglePutMessageTopicSizeTotal(String topic) {
-        AtomicLong rs = putMessageTopicSizeTotal.get(topic);
+    public LongAdder getSinglePutMessageTopicSizeTotal(String topic) {
+        LongAdder rs = putMessageTopicSizeTotal.get(topic);
         if (null == rs) {
-            rs = new AtomicLong(0);
-            AtomicLong previous = putMessageTopicSizeTotal.putIfAbsent(topic, rs);
+            rs = new LongAdder();
+            LongAdder previous = putMessageTopicSizeTotal.putIfAbsent(topic, rs);
             if (previous != null) {
                 rs = previous;
             }
@@ -555,11 +555,11 @@ public class StoreStatsService extends ServiceThread {
         return rs;
     }
 
-    public AtomicLong getSinglePutMessageTopicTimesTotal(String topic) {
-        AtomicLong rs = putMessageTopicTimesTotal.get(topic);
+    public LongAdder getSinglePutMessageTopicTimesTotal(String topic) {
+        LongAdder rs = putMessageTopicTimesTotal.get(topic);
         if (null == rs) {
-            rs = new AtomicLong(0);
-            AtomicLong previous = putMessageTopicTimesTotal.putIfAbsent(topic, rs);
+            rs = new LongAdder();
+            LongAdder previous = putMessageTopicTimesTotal.putIfAbsent(topic, rs);
             if (previous != null) {
                 rs = previous;
             }
@@ -567,11 +567,11 @@ public class StoreStatsService extends ServiceThread {
         return rs;
     }
 
-    public Map<String, AtomicLong> getPutMessageTopicTimesTotal() {
+    public Map<String, LongAdder> getPutMessageTopicTimesTotal() {
         return putMessageTopicTimesTotal;
     }
 
-    public Map<String, AtomicLong> getPutMessageTopicSizeTotal() {
+    public Map<String, LongAdder> getPutMessageTopicSizeTotal() {
         return putMessageTopicSizeTotal;
     }
 

store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java

@@ -17,10 +17,14 @@
 package org.apache.rocketmq.store.config;
 
 import java.io.File;
+
 import org.apache.rocketmq.common.annotation.ImportantField;
 import org.apache.rocketmq.store.ConsumeQueue;
 
 public class MessageStoreConfig {
+
+    public static final String MULTI_PATH_SPLITTER = System.getProperty("rocketmq.broker.multiPathSplitter", ",");
+
     //The root directory in which the log data is kept
     @ImportantField
     private String storePathRootDir = System.getProperty("user.home") + File.separator + "store";
@@ -30,6 +34,8 @@ public class MessageStoreConfig {
     private String storePathCommitLog = System.getProperty("user.home") + File.separator + "store"
         + File.separator + "commitlog";
 
+    private String readOnlyCommitLogStorePaths = null;
+
     // CommitLog file size,default is 1G
     private int mappedFileSizeCommitLog = 1024 * 1024 * 1024;
     // ConsumeQueue file size,default is 30W
@@ -676,6 +682,13 @@ public class MessageStoreConfig {
         this.commitCommitLogThoroughInterval = commitCommitLogThoroughInterval;
     }
 
+    public String getReadOnlyCommitLogStorePaths() {
+        return readOnlyCommitLogStorePaths;
+    }
+
+    public void setReadOnlyCommitLogStorePaths(String readOnlyCommitLogStorePaths) {
+        this.readOnlyCommitLogStorePaths = readOnlyCommitLogStorePaths;
+    }
     public String getdLegerGroup() {
         return dLegerGroup;
     }

store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java

@@ -502,8 +502,8 @@ public class DLedgerCommitLog extends CommitLog {
             PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult);
             if (putMessageStatus == PutMessageStatus.PUT_OK) {
                 // Statistics
-                storeStatsService.getSinglePutMessageTopicTimesTotal(finalTopic).incrementAndGet();
-                storeStatsService.getSinglePutMessageTopicSizeTotal(msg.getTopic()).addAndGet(appendResult.getWroteBytes());
+                storeStatsService.getSinglePutMessageTopicTimesTotal(finalTopic).add(1);
+                storeStatsService.getSinglePutMessageTopicSizeTotal(msg.getTopic()).add(appendResult.getWroteBytes());
             }
             return putMessageResult;
         });
@@ -629,8 +629,8 @@ public class DLedgerCommitLog extends CommitLog {
             PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult);
             if (putMessageStatus == PutMessageStatus.PUT_OK) {
                 // Statistics
-                storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).incrementAndGet();
-                storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).addAndGet(appendResult.getWroteBytes());
+                storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).add(1);
+                storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).add(appendResult.getWroteBytes());
             }
             return putMessageResult;
         });

store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java

@@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.rocketmq.common.ConfigManager;
+import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicFilterType;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.topic.TopicValidator;
@@ -77,8 +78,7 @@ public class ScheduleMessageService extends ConfigManager {
     }
 
     /**
-     * @param writeMessageStore
-     *     the writeMessageStore to set
+     * @param writeMessageStore the writeMessageStore to set
      */
     public void setWriteMessageStore(MessageStore writeMessageStore) {
         this.writeMessageStore = writeMessageStore;
@@ -132,7 +132,9 @@ public class ScheduleMessageService extends ConfigManager {
                 @Override
                 public void run() {
                     try {
-                        if (started.get()) ScheduleMessageService.this.persist();
+                        if (started.get()) {
+                            ScheduleMessageService.this.persist();
+                        }
                     } catch (Throwable e) {
                         log.error("scheduleAtFixedRate flush exception", e);
                     }
@@ -164,9 +166,46 @@ public class ScheduleMessageService extends ConfigManager {
     public boolean load() {
         boolean result = super.load();
         result = result && this.parseDelayLevel();
+        result = result && this.correctDelayOffset();
         return result;
     }
 
+    public boolean correctDelayOffset() {
+        try {
+            for (int delayLevel : delayLevelTable.keySet()) {
+                ConsumeQueue cq =
+                    ScheduleMessageService.this.defaultMessageStore.findConsumeQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC,
+                        delayLevel2QueueId(delayLevel));
+                Long currentDelayOffset = offsetTable.get(delayLevel);
+                if (currentDelayOffset == null || cq == null) {
+                    continue;
+                }
+                long correctDelayOffset = currentDelayOffset;
+                long cqMinOffset = cq.getMinOffsetInQueue();
+                long cqMaxOffset = cq.getMaxOffsetInQueue();
+                if (currentDelayOffset < cqMinOffset) {
+                    correctDelayOffset = cqMinOffset;
+                    log.error("schedule CQ offset invalid. offset={}, cqMinOffset={}, cqMaxOffset={}, queueId={}",
+                        currentDelayOffset, cqMinOffset, cqMaxOffset, cq.getQueueId());
+                }
+
+                if (currentDelayOffset > cqMaxOffset) {
+                    correctDelayOffset = cqMaxOffset;
+                    log.error("schedule CQ offset invalid. offset={}, cqMinOffset={}, cqMaxOffset={}, queueId={}",
+                        currentDelayOffset, cqMinOffset, cqMaxOffset, cq.getQueueId());
+                }
+                if (correctDelayOffset != currentDelayOffset) {
+                    log.error("correct delay offset [ delayLevel {} ] from {} to {}", delayLevel, currentDelayOffset, correctDelayOffset);
+                    offsetTable.put(delayLevel, correctDelayOffset);
+                }
+            }
+        } catch (Exception e) {
+            log.error("correctDelayOffset exception", e);
+            return false;
+        }
+        return true;
+    }
+
     @Override
     public String configFilePath() {
         return StorePathConfigHelper.getDelayOffsetStorePath(this.defaultMessageStore.getMessageStoreConfig()
@@ -308,7 +347,7 @@ public class ScheduleMessageService extends ConfigManager {
                                         MessageExtBrokerInner msgInner = this.messageTimeup(msgExt);
                                         if (TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC.equals(msgInner.getTopic())) {
                                             log.error("[BUG] the real topic of schedule msg is {}, discard the msg. msg={}",
-                                                    msgInner.getTopic(), msgInner);
+                                                msgInner.getTopic(), msgInner);
                                             continue;
                                         }
                                         PutMessageResult putMessageResult =
@@ -318,6 +357,10 @@ public class ScheduleMessageService extends ConfigManager {
                                         if (putMessageResult != null
                                             && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) {
                                             if (ScheduleMessageService.this.defaultMessageStore.getMessageStoreConfig().isEnableScheduleMessageStats()) {
+                                                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incQueueGetNums(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, delayLevel - 1, putMessageResult.getAppendMessageResult().getMsgNum());
+                                                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incQueueGetSize(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, delayLevel - 1, putMessageResult.getAppendMessageResult().getWroteBytes());
+                                                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incGroupGetNums(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, putMessageResult.getAppendMessageResult().getMsgNum());
+                                                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incGroupGetSize(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, putMessageResult.getAppendMessageResult().getWroteBytes());
                                                 ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1);
                                                 ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(),
                                                     putMessageResult.getAppendMessageResult().getWroteBytes());
@@ -339,14 +382,9 @@ public class ScheduleMessageService extends ConfigManager {
                                     } catch (Exception e) {
                                         /*
                                          * XXX: warn and notify me
-
-
-
                                          */
                                         log.error(
-                                            "ScheduleMessageService, messageTimeup execute error, drop it. msgExt="
-                                                + msgExt + ", nextOffset=" + nextOffset + ",offsetPy="
-                                                + offsetPy + ",sizePy=" + sizePy, e);
+                                            "ScheduleMessageService, messageTimeup execute error, drop it. msgExt={}, nextOffset={}, offsetPy={}, sizePy={}", msgExt, nextOffset, offsetPy, sizePy, e);
                                     }
                                 }
                             } else {
@@ -371,10 +409,17 @@ public class ScheduleMessageService extends ConfigManager {
                 else {
 
                     long cqMinOffset = cq.getMinOffsetInQueue();
+                    long cqMaxOffset = cq.getMaxOffsetInQueue();
                     if (offset < cqMinOffset) {
                         failScheduleOffset = cqMinOffset;
-                        log.error("schedule CQ offset invalid. offset=" + offset + ", cqMinOffset="
-                            + cqMinOffset + ", queueId=" + cq.getQueueId());
+                        log.error("schedule CQ offset invalid. offset={}, cqMinOffset={}, cqMaxOffset={}, queueId={}",
+                            offset, cqMinOffset, cqMaxOffset, cq.getQueueId());
+                    }
+
+                    if (offset > cqMaxOffset) {
+                        failScheduleOffset = cqMaxOffset;
+                        log.error("schedule CQ offset invalid. offset={}, cqMinOffset={}, cqMaxOffset={}, queueId={}",
+                            offset, cqMinOffset, cqMaxOffset, cq.getQueueId());
                     }
                 }
             } // end of if (cq != null)

store/src/main/java/org/apache/rocketmq/store/stats/BrokerStats.java

@@ -45,7 +45,7 @@ public class BrokerStats {
         this.msgPutTotalTodayMorning =
             this.defaultMessageStore.getStoreStatsService().getPutMessageTimesTotal();
         this.msgGetTotalTodayMorning =
-            this.defaultMessageStore.getStoreStatsService().getGetMessageTransferedMsgCount().get();
+            this.defaultMessageStore.getStoreStatsService().getGetMessageTransferedMsgCount().longValue();
 
         log.info("yesterday put message total: {}", msgPutTotalTodayMorning - msgPutTotalYesterdayMorning);
         log.info("yesterday get message total: {}", msgGetTotalTodayMorning - msgGetTotalYesterdayMorning);
@@ -88,6 +88,6 @@ public class BrokerStats {
     }
 
     public long getMsgGetTotalTodayNow() {
-        return this.defaultMessageStore.getStoreStatsService().getGetMessageTransferedMsgCount().get();
+        return this.defaultMessageStore.getStoreStatsService().getGetMessageTransferedMsgCount().longValue();
     }
 }

store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java

@@ -29,6 +29,10 @@ import org.apache.rocketmq.common.stats.StatsItemSet;
 
 public class BrokerStatsManager {
 
+    public static final String QUEUE_PUT_NUMS = "QUEUE_PUT_NUMS";
+    public static final String QUEUE_PUT_SIZE = "QUEUE_PUT_SIZE";
+    public static final String QUEUE_GET_NUMS = "QUEUE_GET_NUMS";
+    public static final String QUEUE_GET_SIZE = "QUEUE_GET_SIZE";
     public static final String TOPIC_PUT_NUMS = "TOPIC_PUT_NUMS";
     public static final String TOPIC_PUT_SIZE = "TOPIC_PUT_SIZE";
     public static final String GROUP_GET_NUMS = "GROUP_GET_NUMS";
@@ -74,6 +78,10 @@ public class BrokerStatsManager {
     public BrokerStatsManager(String clusterName) {
         this.clusterName = clusterName;
 
+        this.statsTable.put(QUEUE_PUT_NUMS, new StatsItemSet(QUEUE_PUT_NUMS, this.scheduledExecutorService, log));
+        this.statsTable.put(QUEUE_PUT_SIZE, new StatsItemSet(QUEUE_PUT_SIZE, this.scheduledExecutorService, log));
+        this.statsTable.put(QUEUE_GET_NUMS, new StatsItemSet(QUEUE_GET_NUMS, this.scheduledExecutorService, log));
+        this.statsTable.put(QUEUE_GET_SIZE, new StatsItemSet(QUEUE_GET_SIZE, this.scheduledExecutorService, log));
         this.statsTable.put(TOPIC_PUT_NUMS, new StatsItemSet(TOPIC_PUT_NUMS, this.scheduledExecutorService, log));
         this.statsTable.put(TOPIC_PUT_SIZE, new StatsItemSet(TOPIC_PUT_SIZE, this.scheduledExecutorService, log));
         this.statsTable.put(GROUP_GET_NUMS, new StatsItemSet(GROUP_GET_NUMS, this.scheduledExecutorService, log));
@@ -124,8 +132,12 @@ public class BrokerStatsManager {
     public void onTopicDeleted(final String topic) {
         this.statsTable.get(TOPIC_PUT_NUMS).delValue(topic);
         this.statsTable.get(TOPIC_PUT_SIZE).delValue(topic);
+        this.statsTable.get(QUEUE_PUT_NUMS).delValueByPrefixKey(topic, "@");
+        this.statsTable.get(QUEUE_PUT_SIZE).delValueByPrefixKey(topic, "@");
         this.statsTable.get(GROUP_GET_NUMS).delValueByPrefixKey(topic, "@");
         this.statsTable.get(GROUP_GET_SIZE).delValueByPrefixKey(topic, "@");
+        this.statsTable.get(QUEUE_GET_NUMS).delValueByPrefixKey(topic, "@");
+        this.statsTable.get(QUEUE_GET_SIZE).delValueByPrefixKey(topic, "@");
         this.statsTable.get(SNDBCK_PUT_NUMS).delValueByPrefixKey(topic, "@");
         this.statsTable.get(GROUP_GET_LATENCY).delValueByInfixKey(topic, "@");
         this.momentStatsItemSetFallSize.delValueByInfixKey(topic, "@");
@@ -135,12 +147,36 @@ public class BrokerStatsManager {
     public void onGroupDeleted(final String group) {
         this.statsTable.get(GROUP_GET_NUMS).delValueBySuffixKey(group, "@");
         this.statsTable.get(GROUP_GET_SIZE).delValueBySuffixKey(group, "@");
+        this.statsTable.get(QUEUE_GET_NUMS).delValueBySuffixKey(group, "@");
+        this.statsTable.get(QUEUE_GET_SIZE).delValueBySuffixKey(group, "@");
         this.statsTable.get(SNDBCK_PUT_NUMS).delValueBySuffixKey(group, "@");
         this.statsTable.get(GROUP_GET_LATENCY).delValueBySuffixKey(group, "@");
         this.momentStatsItemSetFallSize.delValueBySuffixKey(group, "@");
         this.momentStatsItemSetFallTime.delValueBySuffixKey(group, "@");
     }
 
+    public void incQueuePutNums(final String topic, final Integer queueId) {
+        this.statsTable.get(QUEUE_PUT_NUMS).addValue(buildStatsKey(topic, String.valueOf(queueId)), 1, 1);
+    }
+
+    public void incQueuePutNums(final String topic, final Integer queueId, int num, int times) {
+        this.statsTable.get(QUEUE_PUT_NUMS).addValue(buildStatsKey(topic, String.valueOf(queueId)), num, times);
+    }
+
+    public void incQueuePutSize(final String topic, final Integer queueId, final int size) {
+        this.statsTable.get(QUEUE_PUT_SIZE).addValue(buildStatsKey(topic, String.valueOf(queueId)), size, 1);
+    }
+
+    public void incQueueGetNums(final String group, final String topic, final Integer queueId, final int incValue) {
+        final String statsKey = buildStatsKey(buildStatsKey(topic, String.valueOf(queueId)), group);
+        this.statsTable.get(QUEUE_GET_NUMS).addValue(statsKey, incValue, 1);
+    }
+
+    public void incQueueGetSize(final String group, final String topic, final Integer queueId, final int incValue) {
+        final String statsKey = buildStatsKey(buildStatsKey(topic, String.valueOf(queueId)), group);
+        this.statsTable.get(QUEUE_GET_SIZE).addValue(statsKey, incValue, 1);
+    }
+
     public void incTopicPutNums(final String topic) {
         this.statsTable.get(TOPIC_PUT_NUMS).addValue(topic, 1, 1);
     }
@@ -158,11 +194,11 @@ public class BrokerStatsManager {
         this.statsTable.get(GROUP_GET_NUMS).addValue(statsKey, incValue, 1);
     }
 
-    public String buildStatsKey(String topic, String group) {
+    public String buildStatsKey(String prefix, String suffix) {
         StringBuffer strBuilder = new StringBuffer();
-        strBuilder.append(topic);
+        strBuilder.append(prefix);
         strBuilder.append("@");
-        strBuilder.append(group);
+        strBuilder.append(suffix);
         return strBuilder.toString();
     }
 
@@ -177,15 +213,15 @@ public class BrokerStatsManager {
     }
 
     public void incBrokerPutNums() {
-        this.statsTable.get(BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().incrementAndGet();
+        this.statsTable.get(BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(1);
     }
 
     public void incBrokerPutNums(final int incValue) {
-        this.statsTable.get(BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().addAndGet(incValue);
+        this.statsTable.get(BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);
     }
 
     public void incBrokerGetNums(final int incValue) {
-        this.statsTable.get(BROKER_GET_NUMS).getAndCreateStatsItem(this.clusterName).getValue().addAndGet(incValue);
+        this.statsTable.get(BROKER_GET_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(incValue);
     }
 
     public void incSendBackNums(final String group, final String topic) {
@@ -217,7 +253,7 @@ public class BrokerStatsManager {
     }
 
     public String buildCommercialStatsKey(String owner, String topic, String group, String type) {
-        StringBuffer strBuilder = new StringBuffer();
+        StringBuilder strBuilder = new StringBuilder();
         strBuilder.append(owner);
         strBuilder.append("@");
         strBuilder.append(topic);

store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreCleanFilesTest.java

@@ -94,6 +94,41 @@ public class DefaultMessageStoreCleanFilesTest {
 
     }
 
+    @Test
+    public void testIsSpaceFullMultiCommitLogStorePath() throws Exception {
+        String deleteWhen = "04";
+        // the min value of diskMaxUsedSpaceRatio.
+        int diskMaxUsedSpaceRatio = 1;
+        // used to  set disk-full flag
+        double diskSpaceCleanForciblyRatio = 0.01D;
+        MessageStoreConfig config = genMessageStoreConfig(deleteWhen, diskMaxUsedSpaceRatio);
+        String storePath = config.getStorePathCommitLog();
+        StringBuilder storePathBuilder = new StringBuilder();
+        for (int i = 0; i < 3; i++) {
+            storePathBuilder.append(storePath).append(i).append(MessageStoreConfig.MULTI_PATH_SPLITTER);
+        }
+        config.setStorePathCommitLog(storePathBuilder.toString());
+        String[] paths = config.getStorePathCommitLog().trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+        assertEquals(3, paths.length);
+        initMessageStore(config, diskSpaceCleanForciblyRatio);
+
+
+
+        // build and put 55 messages, exactly one message per CommitLog file.
+        buildAndPutMessagesToMessageStore(msgCount);
+        MappedFileQueue commitLogQueue = getMappedFileQueueCommitLog();
+        assertEquals(fileCountCommitLog, commitLogQueue.getMappedFiles().size());
+        int fileCountConsumeQueue = getFileCountConsumeQueue();
+        MappedFileQueue consumeQueue = getMappedFileQueueConsumeQueue();
+        assertEquals(fileCountConsumeQueue, consumeQueue.getMappedFiles().size());
+        cleanCommitLogService.isSpaceFull();
+
+        assertEquals(1 << 4, messageStore.getRunningFlags().getFlagBits() & (1 << 4));
+        messageStore.shutdown();
+        messageStore.destroy();
+
+    }
+
     @Test
     public void testIsSpaceFullFunctionFull2Empty() throws Exception {
         String deleteWhen = "04";
@@ -421,6 +456,10 @@ public class DefaultMessageStoreCleanFilesTest {
     }
 
     private void initMessageStore(String deleteWhen, int diskMaxUsedSpaceRatio, double diskSpaceCleanForciblyRatio) throws Exception {
+        initMessageStore(genMessageStoreConfig(deleteWhen,diskMaxUsedSpaceRatio), diskSpaceCleanForciblyRatio);
+    }
+
+    private MessageStoreConfig genMessageStoreConfig(String deleteWhen, int diskMaxUsedSpaceRatio) {
         MessageStoreConfig messageStoreConfig = new MessageStoreConfigForTest();
         messageStoreConfig.setMappedFileSizeCommitLog(mappedFileSize);
         messageStoreConfig.setMappedFileSizeConsumeQueue(mappedFileSize);
@@ -442,7 +481,10 @@ public class DefaultMessageStoreCleanFilesTest {
         String storePathCommitLog = storePathRootDir + File.separator + "commitlog";
         messageStoreConfig.setStorePathRootDir(storePathRootDir);
         messageStoreConfig.setStorePathCommitLog(storePathCommitLog);
+        return messageStoreConfig;
+    }
 
+    private void initMessageStore(MessageStoreConfig messageStoreConfig, double diskSpaceCleanForciblyRatio) throws Exception {
         messageStore = new DefaultMessageStore(messageStoreConfig,
                 new BrokerStatsManager("test"), new MyMessageArrivingListener(), new BrokerConfig());
 

store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java

@@ -605,6 +605,22 @@ public class DefaultMessageStoreTest {
         }
     }
 
+    @Test
+    public void testStorePathOK() {
+        if (messageStore instanceof DefaultMessageStore) {
+            assertTrue(fileExists(((DefaultMessageStore) messageStore).getStorePathPhysic()));
+            assertTrue(fileExists(((DefaultMessageStore) messageStore).getStorePathLogic()));
+        }
+    }
+
+    private boolean fileExists(String path) {
+        if (path != null) {
+            File f = new File(path);
+            return f.exists();
+        }
+        return false;
+    }
+
     private void damageCommitlog(long offset) throws Exception {
         MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
         File file = new File(messageStoreConfig.getStorePathCommitLog() + File.separator + "00000000000000000000");

store/src/test/java/org/apache/rocketmq/store/MultiPathMappedFileQueueTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Test;
+
+
+public class MultiPathMappedFileQueueTest {
+
+    @Test
+    public void testGetLastMappedFile() {
+        final byte[] fixedMsg = new byte[1024];
+
+        MessageStoreConfig config = new MessageStoreConfig();
+        config.setStorePathCommitLog("target/unit_test_store/a/" + MessageStoreConfig.MULTI_PATH_SPLITTER
+                + "target/unit_test_store/b/" + MessageStoreConfig.MULTI_PATH_SPLITTER
+                + "target/unit_test_store/c/");
+        MappedFileQueue mappedFileQueue = new MultiPathMappedFileQueue(config, 1024, null, null);
+        String[] storePaths = config.getStorePathCommitLog().trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+        for (int i = 0; i < 1024; i++) {
+            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * i);
+            assertThat(mappedFile).isNotNull();
+            assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();
+            int idx = i % storePaths.length;
+            assertThat(mappedFile.getFileName().startsWith(storePaths[idx])).isTrue();
+        }
+        mappedFileQueue.shutdown(1000);
+        mappedFileQueue.destroy();
+    }
+
+    @Test
+    public void testLoadReadOnlyMappedFiles() {
+        {
+            //create old mapped files
+            final byte[] fixedMsg = new byte[1024];
+            MessageStoreConfig config = new MessageStoreConfig();
+            config.setStorePathCommitLog("target/unit_test_store/a/" + MessageStoreConfig.MULTI_PATH_SPLITTER
+                    + "target/unit_test_store/b/" + MessageStoreConfig.MULTI_PATH_SPLITTER
+                    + "target/unit_test_store/c/");
+            MappedFileQueue mappedFileQueue = new MultiPathMappedFileQueue(config, 1024, null, null);
+            String[] storePaths = config.getStorePathCommitLog().trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+            for (int i = 0; i < 1024; i++) {
+                MappedFile mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * i);
+                assertThat(mappedFile).isNotNull();
+                assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();
+                int idx = i % storePaths.length;
+                assertThat(mappedFile.getFileName().startsWith(storePaths[idx])).isTrue();
+            }
+            mappedFileQueue.shutdown(1000);
+        }
+
+        // test load and readonly
+        MessageStoreConfig config = new MessageStoreConfig();
+        config.setStorePathCommitLog("target/unit_test_store/b/");
+        config.setReadOnlyCommitLogStorePaths("target/unit_test_store/a" + MessageStoreConfig.MULTI_PATH_SPLITTER
+                + "target/unit_test_store/c");
+        MultiPathMappedFileQueue mappedFileQueue = new MultiPathMappedFileQueue(config, 1024, null, null);
+
+        mappedFileQueue.load();
+
+        assertThat(mappedFileQueue.mappedFiles.size()).isEqualTo(1024);
+        for (int i = 0; i < 1024; i++) {
+            assertThat(mappedFileQueue.mappedFiles.get(i).getFile().getName())
+                    .isEqualTo(UtilAll.offset2FileName(1024 * i));
+        }
+        mappedFileQueue.destroy();
+
+    }
+
+    @Test
+    public void testUpdatePathsOnline() {
+        final byte[] fixedMsg = new byte[1024];
+
+        MessageStoreConfig config = new MessageStoreConfig();
+        config.setStorePathCommitLog("target/unit_test_store/a/" + MessageStoreConfig.MULTI_PATH_SPLITTER
+                + "target/unit_test_store/b/" + MessageStoreConfig.MULTI_PATH_SPLITTER
+                + "target/unit_test_store/c/");
+        MappedFileQueue mappedFileQueue = new MultiPathMappedFileQueue(config, 1024, null, null);
+        String[] storePaths = config.getStorePathCommitLog().trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+        for (int i = 0; i < 1024; i++) {
+            MappedFile mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * i);
+            assertThat(mappedFile).isNotNull();
+            assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();
+            int idx = i % storePaths.length;
+            assertThat(mappedFile.getFileName().startsWith(storePaths[idx])).isTrue();
+
+            if (i == 500) {
+                config.setStorePathCommitLog("target/unit_test_store/a/" + MessageStoreConfig.MULTI_PATH_SPLITTER
+                        + "target/unit_test_store/b/");
+                storePaths = config.getStorePathCommitLog().trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+            }
+        }
+        mappedFileQueue.shutdown(1000);
+        mappedFileQueue.destroy();
+    }
+
+    @Test
+    public void testFullStorePath() {
+        final byte[] fixedMsg = new byte[1024];
+
+        Set<String> fullStorePath = new HashSet<>();
+        MessageStoreConfig config = new MessageStoreConfig();
+        config.setStorePathCommitLog("target/unit_test_store/a/" + MessageStoreConfig.MULTI_PATH_SPLITTER
+                + "target/unit_test_store/b/" + MessageStoreConfig.MULTI_PATH_SPLITTER
+                + "target/unit_test_store/c/");
+        MappedFileQueue mappedFileQueue = new MultiPathMappedFileQueue(config, 1024, null, () -> fullStorePath);
+        String[] storePaths = config.getStorePathCommitLog().trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+        assertThat(storePaths.length).isEqualTo(3);
+
+        MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0);
+        assertThat(mappedFile).isNotNull();
+        assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();
+        assertThat(mappedFile.getFileName().startsWith(storePaths[0])).isTrue();
+
+        mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length);
+        assertThat(mappedFile.getFileName().startsWith(storePaths[1])).isTrue();
+        assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();
+        mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * 2);
+        assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();
+        assertThat(mappedFile.getFileName().startsWith(storePaths[2])).isTrue();
+
+        fullStorePath.add("target/unit_test_store/b/");
+        mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * 3);
+        assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();
+        assertThat(mappedFile.getFileName().startsWith(storePaths[2])).isTrue();
+
+        mappedFile = mappedFileQueue.getLastMappedFile(fixedMsg.length * 4);
+        assertThat(mappedFile.appendMessage(fixedMsg)).isTrue();
+        assertThat(mappedFile.getFileName().startsWith(storePaths[0])).isTrue();
+
+        mappedFileQueue.shutdown(1000);
+        mappedFileQueue.destroy();
+    }
+}
\ No newline at end of file

store/src/test/java/org/apache/rocketmq/store/ScheduleMessageServiceTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store;
+
+import java.lang.reflect.Field;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.store.config.FlushDiskType;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.schedule.ScheduleMessageService;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class ScheduleMessageServiceTest {
+
+    private Random random = new Random();
+
+    @Test
+    public void testCorrectDelayOffset_whenInit() throws Exception {
+
+        ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable = null;
+
+        ScheduleMessageService scheduleMessageService = new ScheduleMessageService((DefaultMessageStore) buildMessageStore());
+        scheduleMessageService.parseDelayLevel();
+
+        ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable1 = new ConcurrentHashMap<>();
+        for (int i = 1; i <= 18; i++) {
+            offsetTable1.put(i, random.nextLong());
+        }
+
+        Field field = scheduleMessageService.getClass().getDeclaredField("offsetTable");
+        field.setAccessible(true);
+        field.set(scheduleMessageService, offsetTable1);
+
+        String jsonStr = scheduleMessageService.encode();
+        scheduleMessageService.decode(jsonStr);
+
+        offsetTable = (ConcurrentMap<Integer, Long>) field.get(scheduleMessageService);
+
+        for (Map.Entry<Integer, Long> entry : offsetTable.entrySet()) {
+            assertEquals(entry.getValue(), offsetTable1.get(entry.getKey()));
+        }
+
+        scheduleMessageService.correctDelayOffset();
+
+        offsetTable = (ConcurrentMap<Integer, Long>) field.get(scheduleMessageService);
+
+        for (long offset : offsetTable.values()) {
+            assertEquals(offset, 0);
+        }
+
+    }
+
+    private MessageStore buildMessageStore() throws Exception {
+        MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
+        messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10);
+        messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10);
+        messageStoreConfig.setMaxHashSlotNum(10000);
+        messageStoreConfig.setMaxIndexNum(100 * 100);
+        messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH);
+        messageStoreConfig.setFlushIntervalConsumeQueue(1);
+        return new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("simpleTest"), null, new BrokerConfig());
+    }
+}

store/src/test/java/org/apache/rocketmq/store/StoreStatsServiceTest.java

@@ -21,6 +21,8 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.LongAdder;
+
 import org.junit.Test;
 
 public class StoreStatsServiceTest {
@@ -30,7 +32,7 @@ public class StoreStatsServiceTest {
         final StoreStatsService storeStatsService = new StoreStatsService();
         int num = Runtime.getRuntime().availableProcessors() * 2;
         for (int j = 0; j < 100; j++) {
-            final AtomicReference<AtomicLong> reference = new AtomicReference<>(null);
+            final AtomicReference<LongAdder> reference = new AtomicReference<>(null);
             final CountDownLatch latch = new CountDownLatch(num);
             final CyclicBarrier barrier = new CyclicBarrier(num);
             for (int i = 0; i < num; i++) {
@@ -39,9 +41,9 @@ public class StoreStatsServiceTest {
                     public void run() {
                         try {
                             barrier.await();
-                            AtomicLong atomicLong = storeStatsService.getSinglePutMessageTopicSizeTotal("test");
-                            if (reference.compareAndSet(null, atomicLong)) {
-                            } else if (reference.get() != atomicLong) {
+                            LongAdder longAdder = storeStatsService.getSinglePutMessageTopicSizeTotal("test");
+                            if (reference.compareAndSet(null, longAdder)) {
+                            } else if (reference.get() != longAdder) {
                                 throw new RuntimeException("Reference should be same!");
                             }
                         } catch (InterruptedException | BrokenBarrierException e) {
@@ -61,7 +63,7 @@ public class StoreStatsServiceTest {
         final StoreStatsService storeStatsService = new StoreStatsService();
         int num = Runtime.getRuntime().availableProcessors() * 2;
         for (int j = 0; j < 100; j++) {
-            final AtomicReference<AtomicLong> reference = new AtomicReference<>(null);
+            final AtomicReference<LongAdder> reference = new AtomicReference<>(null);
             final CountDownLatch latch = new CountDownLatch(num);
             final CyclicBarrier barrier = new CyclicBarrier(num);
             for (int i = 0; i < num; i++) {
@@ -70,9 +72,9 @@ public class StoreStatsServiceTest {
                     public void run() {
                         try {
                             barrier.await();
-                            AtomicLong atomicLong = storeStatsService.getSinglePutMessageTopicTimesTotal("test");
-                            if (reference.compareAndSet(null, atomicLong)) {
-                            } else if (reference.get() != atomicLong) {
+                            LongAdder longAdder = storeStatsService.getSinglePutMessageTopicTimesTotal("test");
+                            if (reference.compareAndSet(null, longAdder)) {
+                            } else if (reference.get() != longAdder) {
                                 throw new RuntimeException("Reference should be same!");
                             }
                         } catch (InterruptedException | BrokenBarrierException e) {

store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java

@@ -149,9 +149,9 @@ public class ScheduleMessageServiceTest {
         assertThat(messageResult.getStatus()).isEqualTo(GetMessageStatus.FOUND);
 
         // get the stats change
-        assertThat(messageStore.getBrokerStatsManager().getStatsItem(BROKER_PUT_NUMS, brokerConfig.getBrokerClusterName()).getValue().get()).isEqualTo(1);
-        assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_NUMS, topic).getValue().get()).isEqualTo(1L);
-        assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_SIZE, topic).getValue().get()).isEqualTo(messageResult.getBufferTotalSize());
+        assertThat(messageStore.getBrokerStatsManager().getStatsItem(BROKER_PUT_NUMS, brokerConfig.getBrokerClusterName()).getValue().sum()).isEqualTo(1);
+        assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_NUMS, topic).getValue().sum()).isEqualTo(1L);
+        assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_SIZE, topic).getValue().sum()).isEqualTo(messageResult.getBufferTotalSize());
 
         // get the message body
         ByteBuffer byteBuffer = ByteBuffer.allocate(messageResult.getBufferTotalSize());

store/src/test/java/stats/BrokerStatsManagerTest.java

@@ -29,6 +29,10 @@ import static org.apache.rocketmq.store.stats.BrokerStatsManager.GROUP_GET_FALL_
 import static org.apache.rocketmq.store.stats.BrokerStatsManager.GROUP_GET_LATENCY;
 import static org.apache.rocketmq.store.stats.BrokerStatsManager.GROUP_GET_NUMS;
 import static org.apache.rocketmq.store.stats.BrokerStatsManager.GROUP_GET_SIZE;
+import static org.apache.rocketmq.store.stats.BrokerStatsManager.QUEUE_GET_NUMS;
+import static org.apache.rocketmq.store.stats.BrokerStatsManager.QUEUE_GET_SIZE;
+import static org.apache.rocketmq.store.stats.BrokerStatsManager.QUEUE_PUT_NUMS;
+import static org.apache.rocketmq.store.stats.BrokerStatsManager.QUEUE_PUT_SIZE;
 import static org.apache.rocketmq.store.stats.BrokerStatsManager.SNDBCK_PUT_NUMS;
 import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_NUMS;
 import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_SIZE;
@@ -38,6 +42,7 @@ public class BrokerStatsManagerTest {
     private BrokerStatsManager brokerStatsManager;
 
     private String TOPIC = "TOPIC_TEST";
+    private Integer QUEUE_ID = 0;
     private String GROUP_NAME = "GROUP_TEST";
 
     @Before
@@ -56,6 +61,36 @@ public class BrokerStatsManagerTest {
         assertThat(brokerStatsManager.getStatsItem("TEST", "TEST")).isNull();
     }
 
+    @Test
+    public void testIncQueuePutNums() {
+        brokerStatsManager.incQueuePutNums(TOPIC, QUEUE_ID);
+        String statsKey = brokerStatsManager.buildStatsKey(TOPIC, String.valueOf(QUEUE_ID));
+        assertThat(brokerStatsManager.getStatsItem(QUEUE_PUT_NUMS, statsKey).getTimes().doubleValue()).isEqualTo(1L);
+        brokerStatsManager.incQueuePutNums(TOPIC, QUEUE_ID, 2, 2);
+        assertThat(brokerStatsManager.getStatsItem(QUEUE_PUT_NUMS, statsKey).getValue().doubleValue()).isEqualTo(3L);
+    }
+
+    @Test
+    public void testIncQueuePutSize() {
+        brokerStatsManager.incQueuePutSize(TOPIC, QUEUE_ID, 2);
+        String statsKey = brokerStatsManager.buildStatsKey(TOPIC, String.valueOf(QUEUE_ID));
+        assertThat(brokerStatsManager.getStatsItem(QUEUE_PUT_SIZE, statsKey).getValue().doubleValue()).isEqualTo(2L);
+    }
+
+    @Test
+    public void testIncQueueGetNums() {
+        brokerStatsManager.incQueueGetNums(GROUP_NAME, TOPIC, QUEUE_ID, 1);
+        final String statsKey = brokerStatsManager.buildStatsKey(brokerStatsManager.buildStatsKey(TOPIC, String.valueOf(QUEUE_ID)), GROUP_NAME);
+        assertThat(brokerStatsManager.getStatsItem(QUEUE_GET_NUMS, statsKey).getValue().doubleValue()).isEqualTo(1L);
+    }
+
+    @Test
+    public void testIncQueueGetSize() {
+        brokerStatsManager.incQueueGetSize(GROUP_NAME, TOPIC, QUEUE_ID, 1);
+        final String statsKey = brokerStatsManager.buildStatsKey(brokerStatsManager.buildStatsKey(TOPIC, String.valueOf(QUEUE_ID)), GROUP_NAME);
+        assertThat(brokerStatsManager.getStatsItem(QUEUE_GET_SIZE, statsKey).getValue().doubleValue()).isEqualTo(1L);
+    }
+
     @Test
     public void testIncTopicPutNums() {
         brokerStatsManager.incTopicPutNums(TOPIC);
@@ -101,8 +136,12 @@ public class BrokerStatsManagerTest {
     public void testOnTopicDeleted() {
         brokerStatsManager.incTopicPutNums(TOPIC);
         brokerStatsManager.incTopicPutSize(TOPIC, 100);
+        brokerStatsManager.incQueuePutNums(TOPIC, QUEUE_ID);
+        brokerStatsManager.incQueuePutSize(TOPIC, QUEUE_ID, 100);
         brokerStatsManager.incGroupGetNums(GROUP_NAME, TOPIC, 1);
         brokerStatsManager.incGroupGetSize(GROUP_NAME, TOPIC, 100);
+        brokerStatsManager.incQueueGetNums(GROUP_NAME, TOPIC, QUEUE_ID, 1);
+        brokerStatsManager.incQueueGetSize(GROUP_NAME, TOPIC, QUEUE_ID, 100);
         brokerStatsManager.incSendBackNums(GROUP_NAME, TOPIC);
         brokerStatsManager.incGroupGetLatency(GROUP_NAME, TOPIC, 1, 1);
         brokerStatsManager.recordDiskFallBehindTime(GROUP_NAME, TOPIC, 1, 11L);
@@ -112,8 +151,12 @@ public class BrokerStatsManagerTest {
 
         Assert.assertNull(brokerStatsManager.getStatsItem(TOPIC_PUT_NUMS, TOPIC));
         Assert.assertNull(brokerStatsManager.getStatsItem(TOPIC_PUT_SIZE, TOPIC));
+        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_PUT_NUMS, TOPIC + "@" + QUEUE_ID));
+        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_PUT_SIZE, TOPIC + "@" + QUEUE_ID));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_SIZE, TOPIC + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_NUMS, TOPIC + "@" + GROUP_NAME));
+        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_GET_SIZE, TOPIC + "@" + QUEUE_ID + "@" + GROUP_NAME));
+        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_GET_NUMS, TOPIC + "@" + QUEUE_ID + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(SNDBCK_PUT_NUMS, TOPIC + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_LATENCY, "1@" + TOPIC + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_SIZE, "1@" + TOPIC + "@" + GROUP_NAME));
@@ -124,6 +167,8 @@ public class BrokerStatsManagerTest {
     public void testOnGroupDeleted(){
         brokerStatsManager.incGroupGetNums(GROUP_NAME, TOPIC, 1);
         brokerStatsManager.incGroupGetSize(GROUP_NAME, TOPIC, 100);
+        brokerStatsManager.incQueueGetNums(GROUP_NAME, TOPIC, QUEUE_ID, 1);
+        brokerStatsManager.incQueueGetSize(GROUP_NAME, TOPIC, QUEUE_ID, 100);
         brokerStatsManager.incSendBackNums(GROUP_NAME, TOPIC);
         brokerStatsManager.incGroupGetLatency(GROUP_NAME, TOPIC, 1, 1);
         brokerStatsManager.recordDiskFallBehindTime(GROUP_NAME, TOPIC, 1, 11L);
@@ -133,6 +178,8 @@ public class BrokerStatsManagerTest {
 
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_SIZE, TOPIC + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_NUMS, TOPIC + "@" + GROUP_NAME));
+        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_GET_SIZE, TOPIC + "@" + QUEUE_ID + "@" + GROUP_NAME));
+        Assert.assertNull(brokerStatsManager.getStatsItem(QUEUE_GET_NUMS, TOPIC + "@" + QUEUE_ID + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(SNDBCK_PUT_NUMS, TOPIC + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_LATENCY, "1@" + TOPIC + "@" + GROUP_NAME));
         Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_SIZE, "1@" + TOPIC + "@" + GROUP_NAME));

test/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>rocketmq-all</artifactId>
         <groupId>org.apache.rocketmq</groupId>
-        <version>4.9.2-SNAPSHOT</version>
+        <version>4.9.3-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>