diff --git a/README.md b/README.md index 33b42800cf31d33ed3e49177a33e382dfda97de1..a47efbcea346022445452299a0a0ea778a8a2438 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,26 @@ -## Apache RocketMQ [![Build Status](https://travis-ci.org/apache/rocketmq.svg?branch=master)](https://travis-ci.org/apache/rocketmq) [![Coverage Status](https://coveralls.io/repos/github/apache/rocketmq/badge.svg?branch=master)](https://coveralls.io/github/apache/rocketmq?branch=master) +## Apache RocketMQ +[![Build Status](https://travis-ci.org/apache/rocketmq.svg?branch=master)](https://travis-ci.org/apache/rocketmq) [![Coverage Status](https://coveralls.io/repos/github/apache/rocketmq/badge.svg?branch=master)](https://coveralls.io/github/apache/rocketmq?branch=master) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.rocketmq/rocketmq-all/badge.svg)](http://search.maven.org/#search%7Cga%7C1%7Corg.apache.rocketmq) [![GitHub release](https://img.shields.io/badge/release-download-orange.svg)](https://rocketmq.apache.org/dowloading/releases) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) +[![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/apache/rocketmq.svg)](http://isitmaintained.com/project/apache/rocketmq "Average time to resolve an issue") +[![Percentage of issues still open](http://isitmaintained.com/badge/open/apache/rocketmq.svg)](http://isitmaintained.com/project/apache/rocketmq "Percentage of issues still open") +![Twitter Follow](https://img.shields.io/twitter/follow/ApacheRocketMQ?style=social) **[Apache RocketMQ](https://rocketmq.apache.org) is a distributed messaging and streaming platform with low latency, high performance and reliability, trillion-level capacity and flexible scalability.** It offers a variety of features: -* Pub/Sub messaging model +* Messageing patterns including publish/subscribe, request/reply and streaming * Financial grade transactional message +* Built-in fault tolerance and high availability configuration options base on [DLedger](https://github.com/openmessaging/openmessaging-storage-dledger) * A variety of cross language clients, such as Java, C/C++, Python, Go * Pluggable transport protocols, such as TCP, SSL, AIO -* Inbuilt message tracing capability, also support opentracing +* Built-in message tracing capability, also support opentracing * Versatile big-data and streaming ecosytem integration * Message retroactivity by time or offset * Reliable FIFO and strict ordered messaging in the same queue -* Efficient pull&push consumption model +* Efficient pull and push consumption model * Million-level message accumulation capacity in a single queue * Multiple messaging protocols like JMS and OpenMessaging * Flexible distributed scale-out deployment architecture @@ -23,7 +28,8 @@ It offers a variety of features: * Various message filter mechanics such as SQL and Tag * Docker images for isolated testing and cloud isolated clusters * Feature-rich administrative dashboard for configuration, metrics and monitoring -* Authentication and authorisation +* Authentication and authorization +* Free open source connectors, for both sources and sinks ---------- diff --git a/acl/pom.xml b/acl/pom.xml index cc995de26165d67c758db06fea717df01d4c8463..74635a4fe5a98b4e40da0568a845e58978730439 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 rocketmq-acl rocketmq-acl ${project.version} @@ -67,6 +67,10 @@ logback-core test + + commons-validator + commons-validator + diff --git a/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java b/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java index b87cc2fa44bc9e93a34020d189bd8d084fa24401..da53e982e07fe9a617ae59ed9eed249744d32987 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.acl; import java.util.List; +import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; @@ -66,4 +67,10 @@ public interface AccessValidator { * @return */ boolean updateGlobalWhiteAddrsConfig(List globalWhiteAddrsList); + + /** + * get broker cluster acl config information + * @return + */ + AclConfig getAllAclConfig(); } diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java index 20e1cfa26b90916aecfec39e2669179c85854e20..8973320237c684c68864c29ff0c4a90d75bfdd3e 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java @@ -23,6 +23,7 @@ import java.io.FileNotFoundException; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; +import java.util.ArrayList; import java.util.Map; import java.util.SortedMap; import org.apache.commons.lang3.StringUtils; @@ -69,24 +70,75 @@ public class AclUtils { return signature; } + public static void IPv6AddressCheck(String netaddress) { + if (isAsterisk(netaddress) || isMinus(netaddress)) { + int asterisk = netaddress.indexOf("*"); + int minus = netaddress.indexOf("-"); +// '*' must be the end of netaddress if it exists + if (asterisk > -1 && asterisk != netaddress.length() - 1) { + throw new AclException(String.format("Netaddress examine scope Exception netaddress is %s", netaddress)); + } + +// format like "2::ac5:78:1-200:*" or "2::ac5:78:1-200" is legal + if (minus > -1) { + if (asterisk == -1) { + if (minus <= netaddress.lastIndexOf(":")) { + throw new AclException(String.format("Netaddress examine scope Exception netaddress is %s", netaddress)); + } + } else { + if (minus <= netaddress.lastIndexOf(":", netaddress.lastIndexOf(":") - 1)) { + throw new AclException(String.format("Netaddress examine scope Exception netaddress is %s", netaddress)); + } + } + } + } + } + + public static String v6ipProcess(String netaddress, String[] strArray, int index) { + int part; + String subAddress; + boolean isAsterisk = isAsterisk(netaddress); + boolean isMinus = isMinus(netaddress); + if (isAsterisk && isMinus) { + part = 6; + int lastColon = netaddress.lastIndexOf(':'); + int secondLastColon = netaddress.substring(0, lastColon).lastIndexOf(':'); + subAddress = netaddress.substring(0, secondLastColon); + } else if (!isAsterisk && !isMinus) { + part = 8; + subAddress = netaddress; + } else { + part = 7; + subAddress = netaddress.substring(0, netaddress.lastIndexOf(':')); + } + return expandIP(subAddress, part); + } + public static void verify(String netaddress, int index) { if (!AclUtils.isScope(netaddress, index)) { throw new AclException(String.format("Netaddress examine scope Exception netaddress is %s", netaddress)); } } - public static String[] getAddreeStrArray(String netaddress, String four) { - String[] fourStrArray = StringUtils.split(four.substring(1, four.length() - 1), ","); + public static String[] getAddreeStrArray(String netaddress, String partialAddress) { + String[] parAddStrArray = StringUtils.split(partialAddress.substring(1, partialAddress.length() - 1), ","); String address = netaddress.substring(0, netaddress.indexOf("{")); - String[] addreeStrArray = new String[fourStrArray.length]; - for (int i = 0; i < fourStrArray.length; i++) { - addreeStrArray[i] = address + fourStrArray[i]; + String[] addreeStrArray = new String[parAddStrArray.length]; + for (int i = 0; i < parAddStrArray.length; i++) { + addreeStrArray[i] = address + parAddStrArray[i]; } return addreeStrArray; } - public static boolean isScope(String num, int index) { - String[] strArray = StringUtils.split(num, "."); + public static boolean isScope(String netaddress, int index) { +// IPv6 Address + if (isColon(netaddress)) { + netaddress = expandIP(netaddress, 8); + String[] strArray = StringUtils.split(netaddress, ":"); + return isIPv6Scope(strArray, index); + } + + String[] strArray = StringUtils.split(netaddress, "."); if (strArray.length != 4) { return false; } @@ -107,6 +159,10 @@ public class AclUtils { } + public static boolean isColon(String netaddress) { + return netaddress.indexOf(':') > -1; + } + public static boolean isScope(String num) { return isScope(Integer.valueOf(num.trim())); } @@ -119,7 +175,7 @@ public class AclUtils { return asterisk.indexOf('*') > -1; } - public static boolean isColon(String colon) { + public static boolean isComma(String colon) { return colon.indexOf(',') > -1; } @@ -128,6 +184,88 @@ public class AclUtils { } + public static boolean isIPv6Scope(String[] num, int index) { + for (int i = 0; i < index; i++) { + int value; + try { + value = Integer.parseInt(num[i], 16); + } catch (NumberFormatException e) { + return false; + } + if (!isIPv6Scope(value)) { + return false; + } + } + return true; + } + + public static boolean isIPv6Scope(int num) { + int min = Integer.parseInt("0", 16); + int max = Integer.parseInt("ffff", 16); + return num >= min && num <= max; + } + + public static String expandIP(String netaddress, int part) { + boolean compress = false; + int compressIndex = -1; + String[] strArray = StringUtils.split(netaddress, ":"); + ArrayList indexes = new ArrayList<>(); + for (int i = 0; i < netaddress.length(); i++) { + if (netaddress.charAt(i) == ':') { + if (indexes.size() > 0 && i - indexes.get(indexes.size() - 1) == 1) { + compressIndex = i; + compress = true; + } + indexes.add(i); + } + } + + for (int i = 0; i < strArray.length; i++) { + if (strArray[i].length() < 4) { + strArray[i] = "0000".substring(0, 4 - strArray[i].length()) + strArray[i]; + } + } + + StringBuilder sb = new StringBuilder(); + if (compress) { + int pos = indexes.indexOf(compressIndex); + int index = 0; + if (!netaddress.startsWith(":")) { + for (int i = 0; i < pos; i++) { + sb.append(strArray[index]).append(":"); + index += 1; + } + } + int zeroNum = part - strArray.length; + if (netaddress.endsWith(":")) { + for (int i = 0; i < zeroNum; i++) { + sb.append("0000"); + if (i != zeroNum - 1) { + sb.append(":"); + } + } + } else { + for (int i = 0; i < zeroNum; i++) { + sb.append("0000").append(":"); + } + for (int i = index; i < strArray.length; i++) { + sb.append(strArray[i]); + if (i != strArray.length - 1) { + sb.append(":"); + } + } + } + } else { + for (int i = 0; i < strArray.length; i++) { + sb.append(strArray[i]); + if (i != strArray.length - 1) { + sb.append(":"); + } + } + } + return sb.toString().toUpperCase(); + } + public static T getYamlDataObject(String path, Class clazz) { Yaml yaml = new Yaml(); FileInputStream fis = null; @@ -148,7 +286,7 @@ public class AclUtils { } } - public static boolean writeDataObject(String path, Map dataMap) { + public static boolean writeDataObject(String path, Map dataMap) { Yaml yaml = new Yaml(); PrintWriter pw = null; try { @@ -172,15 +310,15 @@ public class AclUtils { yamlDataObject = AclUtils.getYamlDataObject(fileName, JSONObject.class); } catch (Exception e) { - log.error("Convert yaml file to data object error, ",e); + log.error("Convert yaml file to data object error, ", e); return null; } if (yamlDataObject == null || yamlDataObject.isEmpty()) { - log.warn("Cannot find conf file :{}, acl isn't be enabled." ,fileName); + log.warn("Cannot find conf file :{}, acl isn't be enabled.", fileName); return null; } - + String accessKey = yamlDataObject.getString(AclConstants.CONFIG_ACCESS_KEY); String secretKey = yamlDataObject.getString(AclConstants.CONFIG_SECRET_KEY); @@ -189,7 +327,7 @@ public class AclUtils { return null; } - return new AclClientRPCHook(new SessionCredentials(accessKey,secretKey)); + return new AclClientRPCHook(new SessionCredentials(accessKey, secretKey)); } } diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java index c8ce23908484c1f56c9576c6ee97c7b907227539..7d96d41548c8ebe5b8a410d5c35b255641386b65 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java @@ -26,6 +26,7 @@ import org.apache.rocketmq.acl.common.AclException; import org.apache.rocketmq.acl.common.AclUtils; import org.apache.rocketmq.acl.common.Permission; import org.apache.rocketmq.acl.common.SessionCredentials; +import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; @@ -50,7 +51,7 @@ public class PlainAccessValidator implements AccessValidator { public AccessResource parse(RemotingCommand request, String remoteAddr) { PlainAccessResource accessResource = new PlainAccessResource(); if (remoteAddr != null && remoteAddr.contains(":")) { - accessResource.setWhiteRemoteAddress(remoteAddr.split(":")[0]); + accessResource.setWhiteRemoteAddress(remoteAddr.substring(0, remoteAddr.lastIndexOf(':'))); } else { accessResource.setWhiteRemoteAddress(remoteAddr); } @@ -155,4 +156,7 @@ public class PlainAccessValidator implements AccessValidator { return aclPlugEngine.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList); } + @Override public AclConfig getAllAclConfig() { + return aclPlugEngine.getAllAclConfig(); + } } diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java index fc7f0f3fdb81c8b1257d915f522e544889e11d52..89638f6ac2cdcc818537887fb77851cdb16f0b1e 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java @@ -30,6 +30,7 @@ import org.apache.rocketmq.acl.common.AclConstants; import org.apache.rocketmq.acl.common.AclException; import org.apache.rocketmq.acl.common.AclUtils; import org.apache.rocketmq.acl.common.Permission; +import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.PlainAccessConfig; @@ -270,6 +271,28 @@ public class PlainPermissionManager { return false; } + public AclConfig getAllAclConfig() { + AclConfig aclConfig = new AclConfig(); + List configs = new ArrayList<>(); + List whiteAddrs = new ArrayList<>(); + JSONObject plainAclConfData = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, + JSONObject.class); + if (plainAclConfData == null || plainAclConfData.isEmpty()) { + throw new AclException(String.format("%s file is not data", fileHome + File.separator + fileName)); + } + JSONArray globalWhiteAddrs = plainAclConfData.getJSONArray(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS); + if (globalWhiteAddrs != null && !globalWhiteAddrs.isEmpty()) { + whiteAddrs = globalWhiteAddrs.toJavaList(String.class); + } + JSONArray accounts = plainAclConfData.getJSONArray(AclConstants.CONFIG_ACCOUNTS); + if (accounts != null && !accounts.isEmpty()) { + configs = accounts.toJavaList(PlainAccessConfig.class); + } + aclConfig.setGlobalWhiteAddrs(whiteAddrs); + aclConfig.setPlainAccessConfigs(configs); + return aclConfig; + } + private void watch() { try { String watchFilePath = fileHome + fileName; diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java index cc2dcee77f9b71d5a516b2eb68a10b2744d6364e..6931eb7c426aa3511769f5e9fa8a4abc6c260bb3 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java @@ -19,6 +19,7 @@ package org.apache.rocketmq.acl.plain; import java.util.HashSet; import java.util.Set; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.rocketmq.acl.common.AclException; import org.apache.rocketmq.acl.common.AclUtils; import org.apache.rocketmq.common.constant.LoggerName; @@ -41,17 +42,26 @@ public class RemoteAddressStrategyFactory { if (StringUtils.isBlank(remoteAddr)) { return BLANK_NET_ADDRESS_STRATEGY; } - if ("*".equals(remoteAddr) || "*.*.*.*".equals(remoteAddr)) { + if ("*".equals(remoteAddr) || "*.*.*.*".equals(remoteAddr) || "*:*:*:*:*:*:*:*".equals(remoteAddr)) { return NULL_NET_ADDRESS_STRATEGY; } if (remoteAddr.endsWith("}")) { - String[] strArray = StringUtils.split(remoteAddr, "."); - String four = strArray[3]; - if (!four.startsWith("{")) { - throw new AclException(String.format("MultipleRemoteAddressStrategy netaddress examine scope Exception netaddress", remoteAddr)); + if (AclUtils.isColon(remoteAddr)) { + String[] strArray = StringUtils.split(remoteAddr, ":"); + String last = strArray[strArray.length - 1]; + if (!last.startsWith("{")) { + throw new AclException(String.format("MultipleRemoteAddressStrategy netaddress examine scope Exception netaddress", remoteAddr)); + } + return new MultipleRemoteAddressStrategy(AclUtils.getAddreeStrArray(remoteAddr, last)); + } else { + String[] strArray = StringUtils.split(remoteAddr, "."); + String four = strArray[3]; + if (!four.startsWith("{")) { + throw new AclException(String.format("MultipleRemoteAddressStrategy netaddress examine scope Exception netaddress", remoteAddr)); + } + return new MultipleRemoteAddressStrategy(AclUtils.getAddreeStrArray(remoteAddr, four)); } - return new MultipleRemoteAddressStrategy(AclUtils.getAddreeStrArray(remoteAddr, four)); - } else if (AclUtils.isColon(remoteAddr)) { + } else if (AclUtils.isComma(remoteAddr)) { return new MultipleRemoteAddressStrategy(StringUtils.split(remoteAddr, ",")); } else if (AclUtils.isAsterisk(remoteAddr) || AclUtils.isMinus(remoteAddr)) { return new RangeRemoteAddressStrategy(remoteAddr); @@ -81,15 +91,26 @@ public class RemoteAddressStrategyFactory { private final Set multipleSet = new HashSet<>(); public MultipleRemoteAddressStrategy(String[] strArray) { + InetAddressValidator validator = InetAddressValidator.getInstance(); for (String netaddress : strArray) { - AclUtils.verify(netaddress, 4); - multipleSet.add(netaddress); + if (validator.isValidInet4Address(netaddress)) { + multipleSet.add(netaddress); + } else if (validator.isValidInet6Address(netaddress)) { + multipleSet.add(AclUtils.expandIP(netaddress, 8)); + } else { + throw new AclException(String.format("Netaddress examine Exception netaddress is %s", netaddress)); + } } } @Override public boolean match(PlainAccessResource plainAccessResource) { - return multipleSet.contains(plainAccessResource.getWhiteRemoteAddress()); + InetAddressValidator validator = InetAddressValidator.getInstance(); + String whiteRemoteAddress = plainAccessResource.getWhiteRemoteAddress(); + if (validator.isValidInet6Address(whiteRemoteAddress)) { + whiteRemoteAddress = AclUtils.expandIP(whiteRemoteAddress, 8); + } + return multipleSet.contains(whiteRemoteAddress); } } @@ -100,12 +121,16 @@ public class RemoteAddressStrategyFactory { public OneRemoteAddressStrategy(String netaddress) { this.netaddress = netaddress; - AclUtils.verify(netaddress, 4); + InetAddressValidator validator = InetAddressValidator.getInstance(); + if (!(validator.isValidInet4Address(netaddress) || validator.isValidInet6Address(netaddress))) { + throw new AclException(String.format("Netaddress examine Exception netaddress is %s", netaddress)); + } } @Override public boolean match(PlainAccessResource plainAccessResource) { - return netaddress.equals(plainAccessResource.getWhiteRemoteAddress()); + String writeRemoteAddress = AclUtils.expandIP(plainAccessResource.getWhiteRemoteAddress(), 8).toUpperCase(); + return AclUtils.expandIP(netaddress, 8).toUpperCase().equals(writeRemoteAddress); } } @@ -121,14 +146,29 @@ public class RemoteAddressStrategyFactory { private int index; public RangeRemoteAddressStrategy(String remoteAddr) { - String[] strArray = StringUtils.split(remoteAddr, "."); - if (analysis(strArray, 1) || analysis(strArray, 2) || analysis(strArray, 3)) { - AclUtils.verify(remoteAddr, index - 1); - StringBuffer sb = new StringBuffer().append(strArray[0].trim()).append(".").append(strArray[1].trim()).append("."); - if (index == 3) { - sb.append(strArray[2].trim()).append("."); +// IPv6 Address + if (AclUtils.isColon(remoteAddr)) { + AclUtils.IPv6AddressCheck(remoteAddr); + String[] strArray = StringUtils.split(remoteAddr, ":"); + for (int i = 1; i < strArray.length; i++) { + if (ipv6Analysis(strArray, i)) { + AclUtils.verify(remoteAddr, index - 1); + String preAddress = AclUtils.v6ipProcess(remoteAddr, strArray, index); + this.index = StringUtils.split(preAddress, ":").length; + this.head = preAddress; + break; + } + } + } else { + String[] strArray = StringUtils.split(remoteAddr, "."); + if (analysis(strArray, 1) || analysis(strArray, 2) || analysis(strArray, 3)) { + AclUtils.verify(remoteAddr, index - 1); + StringBuffer sb = new StringBuffer(); + for (int j = 0; j < index; j++) { + sb.append(strArray[j].trim()).append("."); + } + this.head = sb.toString(); } - this.head = sb.toString(); } } @@ -152,6 +192,27 @@ public class RemoteAddressStrategyFactory { return this.end > 0 ? true : false; } + private boolean ipv6Analysis(String[] strArray, int index) { + String value = strArray[index].trim(); + this.index = index; + if ("*".equals(value)) { + int min = Integer.parseInt("0", 16); + int max = Integer.parseInt("ffff", 16); + setValue(min, max); + } else if (AclUtils.isMinus(value)) { + if (value.indexOf("-") == 0) { + throw new AclException(String.format("RangeRemoteAddressStrategy netaddress examine scope Exception value %s ", value)); + } + String[] valueArray = StringUtils.split(value, "-"); + this.start = Integer.parseInt(valueArray[0], 16); + this.end = Integer.parseInt(valueArray[1], 16); + if (!(AclUtils.isIPv6Scope(end) && AclUtils.isIPv6Scope(start) && start <= end)) { + throw new AclException(String.format("RangeRemoteAddressStrategy netaddress examine scope Exception start is %s , end is %s", start, end)); + } + } + return this.end > 0 ? true : false; + } + private void setValue(int start, int end) { this.start = start; this.end = end; @@ -160,21 +221,33 @@ public class RemoteAddressStrategyFactory { @Override public boolean match(PlainAccessResource plainAccessResource) { String netAddress = plainAccessResource.getWhiteRemoteAddress(); - if (netAddress.startsWith(this.head)) { - String value; - if (index == 3) { - value = netAddress.substring(this.head.length()); - } else { - value = netAddress.substring(this.head.length(), netAddress.lastIndexOf('.')); + InetAddressValidator validator = InetAddressValidator.getInstance(); + if (validator.isValidInet4Address(netAddress)) { + if (netAddress.startsWith(this.head)) { + String value; + if (index == 3) { + value = netAddress.substring(this.head.length()); + } else if (index == 2) { + value = netAddress.substring(this.head.length(), netAddress.lastIndexOf('.')); + } else { + value = netAddress.substring(this.head.length(), netAddress.lastIndexOf('.', netAddress.lastIndexOf('.') - 1)); + } + Integer address = Integer.valueOf(value); + if (address >= this.start && address <= this.end) { + return true; + } } - Integer address = Integer.valueOf(value); - if (address >= this.start && address <= this.end) { - return true; + } else if (validator.isValidInet6Address(netAddress)) { + netAddress = AclUtils.expandIP(netAddress, 8).toUpperCase(); + if (netAddress.startsWith(this.head)) { + String value = netAddress.substring(5 * index, 5 * index + 4); + Integer address = Integer.parseInt(value, 16); + if (address >= this.start && address <= this.end) { + return true; + } } } return false; } - } - } diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java index 5b2627de85dc66f13f1868b7b0c5e014aba3117f..5705b745a0e7472ca7a215468ae20775a53babfb 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java @@ -46,20 +46,35 @@ public class AclUtilsTest { addressList.add("1.1.1.3"); addressList.add("1.1.1.4"); Assert.assertEquals(newAddressList, addressList); + +// IPv6 test + String ipv6Address = "1:ac41:9987::bb22:666:{1,2,3,4}"; + String[] ipv6AddressArray = AclUtils.getAddreeStrArray(ipv6Address, "{1,2,3,4}"); + List newIPv6AddressList = new ArrayList<>(); + for (String a : ipv6AddressArray) { + newIPv6AddressList.add(a); + } + + List ipv6AddressList = new ArrayList<>(); + ipv6AddressList.add("1:ac41:9987::bb22:666:1"); + ipv6AddressList.add("1:ac41:9987::bb22:666:2"); + ipv6AddressList.add("1:ac41:9987::bb22:666:3"); + ipv6AddressList.add("1:ac41:9987::bb22:666:4"); + Assert.assertEquals(newIPv6AddressList, ipv6AddressList); } @Test public void isScopeStringArray() { - String adderss = "12"; + String address = "12"; for (int i = 0; i < 6; i++) { - boolean isScope = AclUtils.isScope(adderss, 4); + boolean isScope = AclUtils.isScope(address, 4); if (i == 3) { Assert.assertTrue(isScope); } else { Assert.assertFalse(isScope); } - adderss = adderss + ".12"; + address = address + ".12"; } } @@ -77,6 +92,25 @@ public class AclUtilsTest { isScope = AclUtils.isScope(adderss, 3); Assert.assertFalse(isScope); +// IPv6 test + adderss = StringUtils.split("1050:0000:0000:0000:0005:0600:300c:326b", ":"); + isScope = AclUtils.isIPv6Scope(adderss, 8); + Assert.assertTrue(isScope); + isScope = AclUtils.isIPv6Scope(adderss, 4); + Assert.assertTrue(isScope); + + adderss = StringUtils.split("1050:9876:0000:0000:0005:akkg:300c:326b", ":"); + isScope = AclUtils.isIPv6Scope(adderss, 8); + Assert.assertFalse(isScope); + isScope = AclUtils.isIPv6Scope(adderss, 4); + Assert.assertTrue(isScope); + + adderss = StringUtils.split(AclUtils.expandIP("1050::0005:akkg:300c:326b", 8), ":"); + isScope = AclUtils.isIPv6Scope(adderss, 8); + Assert.assertFalse(isScope); + isScope = AclUtils.isIPv6Scope(adderss, 4); + Assert.assertTrue(isScope); + } @Test @@ -102,6 +136,18 @@ public class AclUtilsTest { isScope = AclUtils.isScope(256); Assert.assertFalse(isScope); + // IPv6 test + int min = Integer.parseInt("0", 16); + int max = Integer.parseInt("ffff", 16); + for (int i = min; i < max + 1; i++) { + isScope = AclUtils.isIPv6Scope(i); + Assert.assertTrue(isScope); + } + isScope = AclUtils.isIPv6Scope(-1); + Assert.assertFalse(isScope); + isScope = AclUtils.isIPv6Scope(max + 1); + Assert.assertFalse(isScope); + } @Test @@ -115,10 +161,10 @@ public class AclUtilsTest { @Test public void isColonTest() { - boolean isColon = AclUtils.isColon(","); + boolean isColon = AclUtils.isComma(","); Assert.assertTrue(isColon); - isColon = AclUtils.isColon("-"); + isColon = AclUtils.isComma("-"); Assert.assertFalse(isColon); } @@ -131,6 +177,36 @@ public class AclUtilsTest { Assert.assertFalse(isMinus); } + @Test + public void v6ipProcessTest() { + String remoteAddr = "5::7:6:1-200:*"; + String[] strArray = StringUtils.split(remoteAddr, ":"); + Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0000:0000:0000:0007:0006"); + + remoteAddr = "5::7:6:1-200"; + strArray = StringUtils.split(remoteAddr, ":"); + Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0000:0000:0000:0000:0007:0006"); + + remoteAddr = "5::7:6:*"; + strArray = StringUtils.split(remoteAddr, ":"); + Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0000:0000:0000:0000:0007:0006"); + + remoteAddr = "5:7:6:*"; + strArray = StringUtils.split(remoteAddr, ":"); + Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0007:0006"); + } + + @Test + public void expandIPTest() { + Assert.assertEquals(AclUtils.expandIP("::1", 8), "0000:0000:0000:0000:0000:0000:0000:0001"); + Assert.assertEquals(AclUtils.expandIP("3::", 8), "0003:0000:0000:0000:0000:0000:0000:0000"); + Assert.assertEquals(AclUtils.expandIP("2::2", 8), "0002:0000:0000:0000:0000:0000:0000:0002"); + Assert.assertEquals(AclUtils.expandIP("4::aac4:92", 8), "0004:0000:0000:0000:0000:0000:AAC4:0092"); + Assert.assertEquals(AclUtils.expandIP("ab23:56:901a::cc6:765:bb:9011", 8), "AB23:0056:901A:0000:0CC6:0765:00BB:9011"); + Assert.assertEquals(AclUtils.expandIP("ab23:56:901a:1:cc6:765:bb:9011", 8), "AB23:0056:901A:0001:0CC6:0765:00BB:9011"); + Assert.assertEquals(AclUtils.expandIP("5::7:6", 6), "0005:0000:0000:0000:0007:0006"); + } + @SuppressWarnings("unchecked") @Test public void getYamlDataObjectTest() { @@ -140,7 +216,7 @@ public class AclUtilsTest { } @Test - public void writeDataObject2YamlFileTest() throws IOException{ + public void writeDataObject2YamlFileTest() throws IOException { String targetFileName = "src/test/resources/conf/plain_write_acl.yml"; File transport = new File(targetFileName); @@ -153,7 +229,7 @@ public class AclUtilsTest { List globalWhiteRemoteAddrs = new ArrayList(); globalWhiteRemoteAddrs.add("10.10.103.*"); globalWhiteRemoteAddrs.add("192.168.0.*"); - aclYamlMap.put("globalWhiteRemoteAddrs",globalWhiteRemoteAddrs); + aclYamlMap.put("globalWhiteRemoteAddrs", globalWhiteRemoteAddrs); // For accounts element in acl yaml config file List> accounts = new ArrayList>(); @@ -166,14 +242,14 @@ public class AclUtilsTest { } }; accounts.add(accountsMap); - aclYamlMap.put("accounts",accounts); + aclYamlMap.put("accounts", accounts); Assert.assertTrue(AclUtils.writeDataObject(targetFileName, aclYamlMap)); transport.delete(); } @Test - public void updateExistedYamlFileTest() throws IOException{ + public void updateExistedYamlFileTest() throws IOException { String targetFileName = "src/test/resources/conf/plain_update_acl.yml"; File transport = new File(targetFileName); @@ -186,7 +262,7 @@ public class AclUtilsTest { List globalWhiteRemoteAddrs = new ArrayList(); globalWhiteRemoteAddrs.add("10.10.103.*"); globalWhiteRemoteAddrs.add("192.168.0.*"); - aclYamlMap.put("globalWhiteRemoteAddrs",globalWhiteRemoteAddrs); + aclYamlMap.put("globalWhiteRemoteAddrs", globalWhiteRemoteAddrs); // Write file to yaml file AclUtils.writeDataObject(targetFileName, aclYamlMap); @@ -201,7 +277,7 @@ public class AclUtilsTest { Map readableMap = AclUtils.getYamlDataObject(targetFileName, Map.class); List updatedGlobalWhiteRemoteAddrs = (List) readableMap.get("globalWhiteRemoteAddrs"); - Assert.assertEquals("192.168.1.2",updatedGlobalWhiteRemoteAddrs.get(0)); + Assert.assertEquals("192.168.1.2", updatedGlobalWhiteRemoteAddrs.get(0)); transport.delete(); } @@ -235,5 +311,4 @@ public class AclUtilsTest { Assert.assertNull(incompleteContRPCHook); } - } diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java index bca90756193a6c57cbb9c4952fcdad80aadb11b9..00b86228c69515cbfbeecbaa20ae3ee855a3e1d0 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java @@ -29,6 +29,7 @@ import org.apache.rocketmq.acl.common.AclConstants; import org.apache.rocketmq.acl.common.AclException; import org.apache.rocketmq.acl.common.AclUtils; import org.apache.rocketmq.acl.common.SessionCredentials; +import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.header.*; @@ -561,4 +562,12 @@ public class PlainAccessValidatorTest { AclUtils.writeDataObject(targetFileName, backUpAclConfigMap); } + @Test + public void getAllAclConfigTest(){ + PlainAccessValidator plainAccessValidator = new PlainAccessValidator(); + AclConfig aclConfig = plainAccessValidator.getAllAclConfig(); + Assert.assertEquals(aclConfig.getGlobalWhiteAddrs().size(), 2); + Assert.assertEquals(aclConfig.getPlainAccessConfigs().size(), 2); + } + } diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java index 2c2e76bb6fdb328be9c15c9f13efa40ff76c04e2..8998dd993ccbcf75c9788fd8cf314c4c93f58771 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java @@ -39,7 +39,7 @@ public class RemoteAddressStrategyTest { plainAccessResource.setWhiteRemoteAddress("*"); RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); Assert.assertEquals(remoteAddressStrategy, RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY); - + plainAccessResource.setWhiteRemoteAddress("*.*.*.*"); remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); Assert.assertEquals(remoteAddressStrategy, RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY); @@ -71,6 +71,35 @@ public class RemoteAddressStrategyTest { plainAccessResource.setWhiteRemoteAddress(""); remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.BlankRemoteAddressStrategy.class); + +// IPv6 test + plainAccessResource.setWhiteRemoteAddress("*:*:*:*:*:*:*:*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy, RemoteAddressStrategyFactory.NULL_NET_ADDRESS_STRATEGY); + + plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:326b"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.OneRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261,1050::0005:0600:300c:3262,1050::0005:0600:300c:3263"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261:{1,2,3}"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.MultipleRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261:1-200"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress("1050:0005:0600:300c:3261:*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class); + + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:3261:1-20:*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + Assert.assertEquals(remoteAddressStrategy.getClass(), RemoteAddressStrategyFactory.RangeRemoteAddressStrategy.class); } @Test(expected = AclException.class) @@ -80,6 +109,8 @@ public class RemoteAddressStrategyTest { remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); plainAccessResource.setWhiteRemoteAddress("256.0.0.1"); remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress("::1ggg"); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); } @Test @@ -94,6 +125,7 @@ public class RemoteAddressStrategyTest { Assert.assertFalse(isMatch); } + @Test public void oneNetaddressStrategyTest() { PlainAccessResource plainAccessResource = new PlainAccessResource(); plainAccessResource.setWhiteRemoteAddress("127.0.0.1"); @@ -109,6 +141,26 @@ public class RemoteAddressStrategyTest { plainAccessResource.setWhiteRemoteAddress("127.0.0.1"); match = remoteAddressStrategy.match(plainAccessResource); Assert.assertTrue(match); + +// Ipv6 test + plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress("::1"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress(""); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertFalse(match); + + plainAccessResource.setWhiteRemoteAddress("::2"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertFalse(match); + + plainAccessResource.setWhiteRemoteAddress("::1"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertTrue(match); + + plainAccessResource.setWhiteRemoteAddress("0000:0000:0000:0000:0000:0000:0000:0001"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertTrue(match); } @Test @@ -122,6 +174,21 @@ public class RemoteAddressStrategyTest { remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); multipleNetaddressStrategyTest(remoteAddressStrategy); + plainAccessResource.setWhiteRemoteAddress("192.100-150.*.*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress("192.130.0.2"); + boolean match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertTrue(match); + + plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1,1050::0005:0600:300c:2,1050::0005:0600:300c:3"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + multipleIPv6NetaddressStrategyTest(remoteAddressStrategy); + + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:{1,2,3}"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + multipleIPv6NetaddressStrategyTest(remoteAddressStrategy); + } @Test(expected = AclException.class) @@ -129,6 +196,8 @@ public class RemoteAddressStrategyTest { PlainAccessResource plainAccessResource = new PlainAccessResource(); plainAccessResource.setWhiteRemoteAddress("127.0.0.1,2,3}"); remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress("::1,2,3}"); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); } private void multipleNetaddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy) { @@ -155,6 +224,30 @@ public class RemoteAddressStrategyTest { } + private void multipleIPv6NetaddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy) { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:1"); + boolean match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertTrue(match); + + plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:2"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertTrue(match); + + plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:3"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertTrue(match); + + plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:4"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertFalse(match); + + plainAccessResource.setWhiteRemoteAddress("1050:0000:0000:0000:0005:0600:300c:0"); + match = remoteAddressStrategy.match(plainAccessResource); + Assert.assertFalse(match); + + } + @Test public void rangeNetaddressStrategyTest() { String head = "127.0.0."; @@ -162,6 +255,7 @@ public class RemoteAddressStrategyTest { plainAccessResource.setWhiteRemoteAddress("127.0.0.1-200"); RemoteAddressStrategy remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); rangeNetaddressStrategyTest(remoteAddressStrategy, head, 1, 200, true); + plainAccessResource.setWhiteRemoteAddress("127.0.0.*"); remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); rangeNetaddressStrategyTest(remoteAddressStrategy, head, 0, 255, true); @@ -169,14 +263,40 @@ public class RemoteAddressStrategyTest { plainAccessResource.setWhiteRemoteAddress("127.0.1-200.*"); remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); rangeNetaddressStrategyThirdlyTest(remoteAddressStrategy, head, 1, 200); - + plainAccessResource.setWhiteRemoteAddress("127.*.*.*"); remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); - rangeNetaddressStrategyThirdlyTest(remoteAddressStrategy, head, 1, 200); - + rangeNetaddressStrategyTest(remoteAddressStrategy, head, 0, 255, true); + plainAccessResource.setWhiteRemoteAddress("127.1-150.*.*"); remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); rangeNetaddressStrategyThirdlyTest(remoteAddressStrategy, head, 1, 200); + +// IPv6 test + head = "1050::0005:0600:300c:"; + plainAccessResource = new PlainAccessResource(); + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1-200"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + rangeIPv6NetaddressStrategyTest(remoteAddressStrategy, head, "1", "200", true); + + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + rangeIPv6NetaddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", true); + + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:3001:*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + rangeIPv6NetaddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", false); + + head = "1050::0005:0600:300c:1:"; + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1-200:*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + rangeIPv6NetaddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", true); + + head = "1050::0005:0600:300c:201:"; + plainAccessResource.setWhiteRemoteAddress("1050::0005:0600:300c:1-200:*"); + remoteAddressStrategy = remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + rangeIPv6NetaddressStrategyTest(remoteAddressStrategy, head, "0", "ffff", false); + } private void rangeNetaddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy, String head, int start, @@ -206,6 +326,25 @@ public class RemoteAddressStrategyTest { } } + private void rangeIPv6NetaddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy, String head, String start, + String end, + boolean isFalse) { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + for (int i = -10; i < 65536 + 100; i++) { + String hex = Integer.toHexString(i); + plainAccessResource.setWhiteRemoteAddress(head + hex); + boolean match = remoteAddressStrategy.match(plainAccessResource); + int startNum = Integer.parseInt(start, 16); + int endNum = Integer.parseInt(end, 16); + if (isFalse && i >= startNum && i <= endNum) { + Assert.assertTrue(match); + continue; + } + Assert.assertFalse(match); + + } + } + @Test(expected = AclException.class) public void rangeNetaddressStrategyExceptionStartGreaterEndTest() { rangeNetaddressStrategyExceptionTest("127.0.0.2-1"); diff --git a/broker/pom.xml b/broker/pom.xml index 2ca7c6abd6484836e9f47088c151ea7dec75d042..511299075800addb0abd2566f632774119742ec6 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index a885cd08989ea00eefc7355c54b99dbe7e8f1547..85009d620f573ab97adb149dd417a86237bf3d92 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -61,6 +61,7 @@ import org.apache.rocketmq.broker.processor.ConsumerManageProcessor; import org.apache.rocketmq.broker.processor.EndTransactionProcessor; import org.apache.rocketmq.broker.processor.PullMessageProcessor; import org.apache.rocketmq.broker.processor.QueryMessageProcessor; +import org.apache.rocketmq.broker.processor.ReplyMessageProcessor; import org.apache.rocketmq.broker.processor.SendMessageProcessor; import org.apache.rocketmq.broker.slave.SlaveSynchronize; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; @@ -132,6 +133,7 @@ public class BrokerController { private final SlaveSynchronize slaveSynchronize; private final BlockingQueue sendThreadPoolQueue; private final BlockingQueue pullThreadPoolQueue; + private final BlockingQueue replyThreadPoolQueue; private final BlockingQueue queryThreadPoolQueue; private final BlockingQueue clientManagerThreadPoolQueue; private final BlockingQueue heartbeatThreadPoolQueue; @@ -147,6 +149,7 @@ public class BrokerController { private TopicConfigManager topicConfigManager; private ExecutorService sendMessageExecutor; private ExecutorService pullMessageExecutor; + private ExecutorService replyMessageExecutor; private ExecutorService queryMessageExecutor; private ExecutorService adminBrokerExecutor; private ExecutorService clientManageExecutor; @@ -194,6 +197,7 @@ public class BrokerController { this.sendThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getSendThreadPoolQueueCapacity()); this.pullThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getPullThreadPoolQueueCapacity()); + this.replyThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getReplyThreadPoolQueueCapacity()); this.queryThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getQueryThreadPoolQueueCapacity()); this.clientManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getClientManagerThreadPoolQueueCapacity()); this.consumerManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getConsumerManagerThreadPoolQueueCapacity()); @@ -277,6 +281,14 @@ public class BrokerController { this.pullThreadPoolQueue, new ThreadFactoryImpl("PullMessageThread_")); + this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.brokerConfig.getProcessReplyMessageThreadPoolNums(), + this.brokerConfig.getProcessReplyMessageThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.replyThreadPoolQueue, + new ThreadFactoryImpl("ProcessReplyMessageThread_")); + this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor( this.brokerConfig.getQueryMessageThreadPoolNums(), this.brokerConfig.getQueryMessageThreadPoolNums(), @@ -553,6 +565,17 @@ public class BrokerController { this.remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, this.pullMessageExecutor); this.pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList); + /** + * ReplyMessageProcessor + */ + ReplyMessageProcessor replyMessageProcessor = new ReplyMessageProcessor(this); + replyMessageProcessor.registerSendMessageHook(sendMessageHookList); + + this.remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor); + /** * QueryMessageProcessor */ @@ -763,6 +786,10 @@ public class BrokerController { this.pullMessageExecutor.shutdown(); } + if (this.replyMessageExecutor != null) { + this.replyMessageExecutor.shutdown(); + } + if (this.adminBrokerExecutor != null) { this.adminBrokerExecutor.shutdown(); } @@ -857,12 +884,9 @@ public class BrokerController { if (!messageStoreConfig.isEnableDLegerCommitLog()) { startProcessorByHa(messageStoreConfig.getBrokerRole()); handleSlaveSynchronize(messageStoreConfig.getBrokerRole()); + this.registerBrokerAll(true, false, true); } - - - this.registerBrokerAll(true, false, true); - this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { @Override diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java index 4b986c0d2497ee0a50a0b10addbfc8c5c33e2548..960b848461db2a11bd4fd2037e9dd7790078c091 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java @@ -178,6 +178,10 @@ public class BrokerStartup { break; } + if (messageStoreConfig.isEnableDLegerCommitLog()) { + brokerConfig.setBrokerId(-1); + } + messageStoreConfig.setHaListenPort(nettyServerConfig.getListenPort() + 1); LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); JoranConfigurator configurator = new JoranConfigurator(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 61ceae53860d16d87045b1af067650f8529424fb..860b3493d1e97c93b178b48e9717aaf328510552 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -17,17 +17,12 @@ package org.apache.rocketmq.broker.client; import io.netty.channel.Channel; - import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - +import java.util.concurrent.ConcurrentHashMap; import org.apache.rocketmq.broker.util.PositiveAtomicCounter; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; @@ -37,190 +32,148 @@ import org.apache.rocketmq.remoting.common.RemotingUtil; public class ProducerManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); - private static final long LOCK_TIMEOUT_MILLIS = 3000; private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; private static final int GET_AVALIABLE_CHANNEL_RETRY_COUNT = 3; - private final Lock groupChannelLock = new ReentrantLock(); - private final HashMap> groupChannelTable = - new HashMap>(); + private final ConcurrentHashMap> groupChannelTable = + new ConcurrentHashMap<>(); + private final ConcurrentHashMap clientChannelTable = new ConcurrentHashMap<>(); private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter(); + public ProducerManager() { } - public HashMap> getGroupChannelTable() { - HashMap> newGroupChannelTable = - new HashMap>(); - try { - if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { - try { - newGroupChannelTable.putAll(groupChannelTable); - } finally { - groupChannelLock.unlock(); - } - } - } catch (InterruptedException e) { - log.error("", e); - } - return newGroupChannelTable; + public ConcurrentHashMap> getGroupChannelTable() { + return groupChannelTable; } public void scanNotActiveChannel() { - try { - if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { - try { - for (final Map.Entry> entry : this.groupChannelTable - .entrySet()) { - final String group = entry.getKey(); - final HashMap chlMap = entry.getValue(); - - Iterator> it = chlMap.entrySet().iterator(); - while (it.hasNext()) { - Entry item = it.next(); - // final Integer id = item.getKey(); - final ClientChannelInfo info = item.getValue(); - - long diff = System.currentTimeMillis() - info.getLastUpdateTimestamp(); - if (diff > CHANNEL_EXPIRED_TIMEOUT) { - it.remove(); - log.warn( - "SCAN: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}", - RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group); - RemotingUtil.closeChannel(info.getChannel()); - } - } - } - } finally { - this.groupChannelLock.unlock(); + for (final Map.Entry> entry : this.groupChannelTable + .entrySet()) { + final String group = entry.getKey(); + final ConcurrentHashMap chlMap = entry.getValue(); + + Iterator> it = chlMap.entrySet().iterator(); + while (it.hasNext()) { + Entry item = it.next(); + // final Integer id = item.getKey(); + final ClientChannelInfo info = item.getValue(); + + long diff = System.currentTimeMillis() - info.getLastUpdateTimestamp(); + if (diff > CHANNEL_EXPIRED_TIMEOUT) { + it.remove(); + clientChannelTable.remove(info.getClientId()); + log.warn( + "SCAN: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}", + RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group); + RemotingUtil.closeChannel(info.getChannel()); } - } else { - log.warn("ProducerManager scanNotActiveChannel lock timeout"); } - } catch (InterruptedException e) { - log.error("", e); } } - public void doChannelCloseEvent(final String remoteAddr, final Channel channel) { + public synchronized void doChannelCloseEvent(final String remoteAddr, final Channel channel) { if (channel != null) { - try { - if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { - try { - for (final Map.Entry> entry : this.groupChannelTable - .entrySet()) { - final String group = entry.getKey(); - final HashMap clientChannelInfoTable = - entry.getValue(); - final ClientChannelInfo clientChannelInfo = - clientChannelInfoTable.remove(channel); - if (clientChannelInfo != null) { - log.info( - "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}", - clientChannelInfo.toString(), remoteAddr, group); - } - - } - } finally { - this.groupChannelLock.unlock(); - } - } else { - log.warn("ProducerManager doChannelCloseEvent lock timeout"); + for (final Map.Entry> entry : this.groupChannelTable + .entrySet()) { + final String group = entry.getKey(); + final ConcurrentHashMap clientChannelInfoTable = + entry.getValue(); + final ClientChannelInfo clientChannelInfo = + clientChannelInfoTable.remove(channel); + if (clientChannelInfo != null) { + clientChannelTable.remove(clientChannelInfo.getClientId()); + log.info( + "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}", + clientChannelInfo.toString(), remoteAddr, group); } - } catch (InterruptedException e) { - log.error("", e); + } } } - public void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) { - try { - ClientChannelInfo clientChannelInfoFound = null; - - if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { - try { - HashMap channelTable = this.groupChannelTable.get(group); - if (null == channelTable) { - channelTable = new HashMap<>(); - this.groupChannelTable.put(group, channelTable); - } - - clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel()); - if (null == clientChannelInfoFound) { - channelTable.put(clientChannelInfo.getChannel(), clientChannelInfo); - log.info("new producer connected, group: {} channel: {}", group, - clientChannelInfo.toString()); - } - } finally { - this.groupChannelLock.unlock(); - } + public synchronized void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) { + ClientChannelInfo clientChannelInfoFound = null; - if (clientChannelInfoFound != null) { - clientChannelInfoFound.setLastUpdateTimestamp(System.currentTimeMillis()); - } - } else { - log.warn("ProducerManager registerProducer lock timeout"); - } - } catch (InterruptedException e) { - log.error("", e); + ConcurrentHashMap channelTable = this.groupChannelTable.get(group); + if (null == channelTable) { + channelTable = new ConcurrentHashMap<>(); + this.groupChannelTable.put(group, channelTable); + } + + clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel()); + if (null == clientChannelInfoFound) { + channelTable.put(clientChannelInfo.getChannel(), clientChannelInfo); + clientChannelTable.put(clientChannelInfo.getClientId(), clientChannelInfo.getChannel()); + log.info("new producer connected, group: {} channel: {}", group, + clientChannelInfo.toString()); + } + + + if (clientChannelInfoFound != null) { + clientChannelInfoFound.setLastUpdateTimestamp(System.currentTimeMillis()); } } - public void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) { - try { - if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { - try { - HashMap channelTable = this.groupChannelTable.get(group); - if (null != channelTable && !channelTable.isEmpty()) { - ClientChannelInfo old = channelTable.remove(clientChannelInfo.getChannel()); - if (old != null) { - log.info("unregister a producer[{}] from groupChannelTable {}", group, - clientChannelInfo.toString()); - } - - if (channelTable.isEmpty()) { - this.groupChannelTable.remove(group); - log.info("unregister a producer group[{}] from groupChannelTable", group); - } - } - } finally { - this.groupChannelLock.unlock(); - } - } else { - log.warn("ProducerManager unregisterProducer lock timeout"); + public synchronized void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) { + ConcurrentHashMap channelTable = this.groupChannelTable.get(group); + if (null != channelTable && !channelTable.isEmpty()) { + ClientChannelInfo old = channelTable.remove(clientChannelInfo.getChannel()); + clientChannelTable.remove(clientChannelInfo.getClientId()); + if (old != null) { + log.info("unregister a producer[{}] from groupChannelTable {}", group, + clientChannelInfo.toString()); + } + + if (channelTable.isEmpty()) { + this.groupChannelTable.remove(group); + log.info("unregister a producer group[{}] from groupChannelTable", group); } - } catch (InterruptedException e) { - log.error("", e); } } public Channel getAvaliableChannel(String groupId) { - HashMap channelClientChannelInfoHashMap = groupChannelTable.get(groupId); + if (groupId == null) { + return null; + } List channelList = new ArrayList(); + ConcurrentHashMap channelClientChannelInfoHashMap = groupChannelTable.get(groupId); if (channelClientChannelInfoHashMap != null) { for (Channel channel : channelClientChannelInfoHashMap.keySet()) { channelList.add(channel); } - int size = channelList.size(); - if (0 == size) { - log.warn("Channel list is empty. groupId={}", groupId); - return null; - } - - int index = positiveAtomicCounter.incrementAndGet() % size; - Channel channel = channelList.get(index); - int count = 0; - boolean isOk = channel.isActive() && channel.isWritable(); - while (count++ < GET_AVALIABLE_CHANNEL_RETRY_COUNT) { - if (isOk) { - return channel; - } - index = (++index) % size; - channel = channelList.get(index); - isOk = channel.isActive() && channel.isWritable(); - } } else { log.warn("Check transaction failed, channel table is empty. groupId={}", groupId); return null; } - return null; + + int size = channelList.size(); + if (0 == size) { + log.warn("Channel list is empty. groupId={}", groupId); + return null; + } + + Channel lastActiveChannel = null; + + int index = positiveAtomicCounter.incrementAndGet() % size; + Channel channel = channelList.get(index); + int count = 0; + boolean isOk = channel.isActive() && channel.isWritable(); + while (count++ < GET_AVALIABLE_CHANNEL_RETRY_COUNT) { + if (isOk) { + return channel; + } + if (channel.isActive()) { + lastActiveChannel = channel; + } + index = (++index) % size; + channel = channelList.get(index); + isOk = channel.isActive() && channel.isWritable(); + } + + return lastActiveChannel; + } + + public Channel findChannel(String clientId) { + return clientChannelTable.get(clientId); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java index b0668d49f87ef166da16d0b37090c0f102772e63..adf027993b5fa64b65f725d6e005b73da8435452 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java @@ -16,16 +16,16 @@ */ package org.apache.rocketmq.broker.processor; +import io.netty.channel.ChannelHandlerContext; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.List; import java.util.Map; import java.util.Random; - -import io.netty.channel.ChannelHandlerContext; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; +import org.apache.rocketmq.broker.topic.TopicValidator; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicFilterType; @@ -171,11 +171,8 @@ public abstract class AbstractSendMessageProcessor implements NettyRequestProces + "] sending message is forbidden"); return response; } - if (!this.brokerController.getTopicConfigManager().isTopicCanSendMessage(requestHeader.getTopic())) { - String errorMsg = "the topic[" + requestHeader.getTopic() + "] is conflict with system reserved words."; - log.warn(errorMsg); - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark(errorMsg); + + if (!TopicValidator.validateTopic(requestHeader.getTopic(), response)) { return response; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 76a051b924c01bcdafd039d34c6c06f844f7063f..b2edc1a20f39cbeb94d4f1356737aac49efc664a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -38,6 +38,8 @@ import org.apache.rocketmq.broker.client.ConsumerGroupInfo; import org.apache.rocketmq.broker.filter.ConsumerFilterData; import org.apache.rocketmq.broker.filter.ExpressionMessageFilter; import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; +import org.apache.rocketmq.broker.topic.TopicValidator; +import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.PlainAccessConfig; @@ -48,12 +50,6 @@ import org.apache.rocketmq.common.admin.OffsetWrapper; import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.admin.TopicStatsTable; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.protocol.header.CreateAccessConfigRequestHeader; -import org.apache.rocketmq.common.protocol.header.DeleteAccessConfigRequestHeader; -import org.apache.rocketmq.common.protocol.header.GetBrokerAclConfigResponseHeader; -import org.apache.rocketmq.common.protocol.header.UpdateGlobalWhiteAddrsConfigRequestHeader; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; @@ -81,10 +77,15 @@ import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; import org.apache.rocketmq.common.protocol.header.CloneGroupOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.common.protocol.header.CreateAccessConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader; +import org.apache.rocketmq.common.protocol.header.DeleteAccessConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.GetAllTopicConfigResponseHeader; +import org.apache.rocketmq.common.protocol.header.GetBrokerAclConfigResponseHeader; +import org.apache.rocketmq.common.protocol.header.GetBrokerClusterAclConfigResponseBody; +import org.apache.rocketmq.common.protocol.header.GetBrokerClusterAclConfigResponseHeader; import org.apache.rocketmq.common.protocol.header.GetBrokerConfigResponseHeader; import org.apache.rocketmq.common.protocol.header.GetConsumeStatsInBrokerHeader; import org.apache.rocketmq.common.protocol.header.GetConsumeStatsRequestHeader; @@ -107,6 +108,7 @@ import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SearchOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.SearchOffsetResponseHeader; +import org.apache.rocketmq.common.protocol.header.UpdateGlobalWhiteAddrsConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.ViewBrokerStatsDataRequestHeader; import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerRequestHeader; import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; @@ -115,6 +117,8 @@ import org.apache.rocketmq.common.stats.StatsItem; import org.apache.rocketmq.common.stats.StatsSnapshot; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.filter.util.BitsArray; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; @@ -226,6 +230,8 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { return updateGlobalWhiteAddrsConfig(ctx, request); case RequestCode.RESUME_CHECK_HALF_MESSAGE: return resumeCheckHalfMessage(ctx, request); + case RequestCode.GET_BROKER_CLUSTER_ACL_CONFIG: + return getBrokerClusterAclConfig(ctx, request); default: break; } @@ -253,6 +259,10 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { return response; } + if (!TopicValidator.validateTopic(requestHeader.getTopic(), response)) { + return response; + } + try { response.setCode(ResponseCode.SUCCESS); response.setOpaque(request.getOpaque()); @@ -307,8 +317,8 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { accessConfig.setWhiteRemoteAddress(requestHeader.getWhiteRemoteAddress()); accessConfig.setDefaultTopicPerm(requestHeader.getDefaultTopicPerm()); accessConfig.setDefaultGroupPerm(requestHeader.getDefaultGroupPerm()); - accessConfig.setTopicPerms(UtilAll.String2List(requestHeader.getTopicPerms(),",")); - accessConfig.setGroupPerms(UtilAll.String2List(requestHeader.getGroupPerms(),",")); + accessConfig.setTopicPerms(UtilAll.string2List(requestHeader.getTopicPerms(), ",")); + accessConfig.setGroupPerms(UtilAll.string2List(requestHeader.getGroupPerms(), ",")); accessConfig.setAdmin(requestHeader.isAdmin()); try { @@ -381,7 +391,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { try { AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class); - if (accessValidator.updateGlobalWhiteAddrsConfig(UtilAll.String2List(requestHeader.getGlobalWhiteAddrs(),","))) { + if (accessValidator.updateGlobalWhiteAddrsConfig(UtilAll.string2List(requestHeader.getGlobalWhiteAddrs(), ","))) { response.setCode(ResponseCode.SUCCESS); response.setOpaque(request.getOpaque()); response.markResponseType(); @@ -428,6 +438,27 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { return null; } + private RemotingCommand getBrokerClusterAclConfig(ChannelHandlerContext ctx, RemotingCommand request) { + + final RemotingCommand response = RemotingCommand.createResponseCommand(GetBrokerClusterAclConfigResponseHeader.class); + + try { + AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class); + GetBrokerClusterAclConfigResponseBody body = new GetBrokerClusterAclConfigResponseBody(); + AclConfig aclConfig = accessValidator.getAllAclConfig(); + body.setGlobalWhiteAddrs(aclConfig.getGlobalWhiteAddrs()); + body.setPlainAccessConfigs(aclConfig.getPlainAccessConfigs()); + response.setCode(ResponseCode.SUCCESS); + response.setBody(body.encode()); + response.setRemark(null); + return response; + } catch (Exception e) { + log.error("Failed to generate a proper getBrokerClusterAclConfig response", e); + } + + return null; + } + private RemotingCommand getAllTopicConfig(ChannelHandlerContext ctx, RemotingCommand request) { final RemotingCommand response = RemotingCommand.createResponseCommand(GetAllTopicConfigResponseHeader.class); // final GetAllTopicConfigResponseHeader responseHeader = @@ -784,7 +815,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { (GetProducerConnectionListRequestHeader) request.decodeCommandCustomHeader(GetProducerConnectionListRequestHeader.class); ProducerConnection bodydata = new ProducerConnection(); - HashMap channelInfoHashMap = + Map channelInfoHashMap = this.brokerController.getProducerManager().getGroupChannelTable().get(requestHeader.getProducerGroup()); if (channelInfoHashMap != null) { Iterator> it = channelInfoHashMap.entrySet().iterator(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java index c9e85ed1a508e9b2850977e3ed23c0637bda68ea..9844cae6c2335df858f3558a69aff15a1aef6562 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java @@ -55,7 +55,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor { final RemotingCommand response = RemotingCommand.createResponseCommand(null); final EndTransactionRequestHeader requestHeader = (EndTransactionRequestHeader)request.decodeCommandCustomHeader(EndTransactionRequestHeader.class); - LOGGER.info("Transaction request:{}", requestHeader); + LOGGER.debug("Transaction request:{}", requestHeader); if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) { response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE); LOGGER.warn("Message store is slave mode, so end transaction is forbidden. "); @@ -132,6 +132,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor { msgInner.setQueueOffset(requestHeader.getTranStateTableOffset()); msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset()); msgInner.setStoreTimestamp(result.getPrepareMessage().getStoreTimestamp()); + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED); RemotingCommand sendResult = sendFinalMessage(msgInner); if (sendResult.getCode() == ResponseCode.SUCCESS) { this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java index 10c0112feb4a48dfbc483a75069f5022625489fc..b4f6daa05eb8fa6a5474a57515e5145c08f91e5f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java @@ -41,8 +41,6 @@ import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.ResponseCode; @@ -52,7 +50,10 @@ import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.topic.OffsetMovedEvent; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.sysflag.PullSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -494,7 +495,21 @@ public class PullMessageProcessor implements NettyRequestProcessor { for (ByteBuffer bb : messageBufferList) { byteBuffer.put(bb); - storeTimestamp = bb.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSTION); + int sysFlag = bb.getInt(MessageDecoder.SYSFLAG_POSITION); +// bornhost has the IPv4 ip if the MessageSysFlag.BORNHOST_V6_FLAG bit of sysFlag is 0 +// IPv4 host = ip(4 byte) + port(4 byte); IPv6 host = ip(16 byte) + port(4 byte) + int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; + int msgStoreTimePos = 4 // 1 TOTALSIZE + + 4 // 2 MAGICCODE + + 4 // 3 BODYCRC + + 4 // 4 QUEUEID + + 4 // 5 FLAG + + 8 // 6 QUEUEOFFSET + + 8 // 7 PHYSICALOFFSET + + 4 // 8 SYSFLAG + + 8 // 9 BORNTIMESTAMP + + bornhostLength; // 10 BORNHOST + storeTimestamp = bb.getLong(msgStoreTimePos); } } finally { getMessageResult.release(); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java new file mode 100644 index 0000000000000000000000000000000000000000..565857a34779c126154694f0ac6b0881d60d5007 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.mqtrace.SendMessageContext; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.ReplyMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2; +import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.stats.BrokerStatsManager; + +public class ReplyMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + + public ReplyMessageProcessor(final BrokerController brokerController) { + super(brokerController); + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + SendMessageContext mqtraceContext = null; + SendMessageRequestHeader requestHeader = parseRequestHeader(request); + if (requestHeader == null) { + return null; + } + + mqtraceContext = buildMsgContext(ctx, requestHeader); + this.executeSendMessageHookBefore(ctx, request, mqtraceContext); + + RemotingCommand response = this.processReplyMessageRequest(ctx, request, mqtraceContext, requestHeader); + + this.executeSendMessageHookAfter(response, mqtraceContext); + return response; + } + + @Override + protected SendMessageRequestHeader parseRequestHeader(RemotingCommand request) throws RemotingCommandException { + SendMessageRequestHeaderV2 requestHeaderV2 = null; + SendMessageRequestHeader requestHeader = null; + switch (request.getCode()) { + case RequestCode.SEND_REPLY_MESSAGE_V2: + requestHeaderV2 = + (SendMessageRequestHeaderV2) request + .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class); + case RequestCode.SEND_REPLY_MESSAGE: + if (null == requestHeaderV2) { + requestHeader = + (SendMessageRequestHeader) request + .decodeCommandCustomHeader(SendMessageRequestHeader.class); + } else { + requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2); + } + default: + break; + } + return requestHeader; + } + + private RemotingCommand processReplyMessageRequest(final ChannelHandlerContext ctx, + final RemotingCommand request, + final SendMessageContext sendMessageContext, + final SendMessageRequestHeader requestHeader) { + final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + + response.setOpaque(request.getOpaque()); + + response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId()); + response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn())); + + log.debug("receive SendReplyMessage request command, {}", request); + final long startTimstamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp(); + if (this.brokerController.getMessageStore().now() < startTimstamp) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimstamp))); + return response; + } + + response.setCode(-1); + super.msgCheck(ctx, requestHeader, response); + if (response.getCode() != -1) { + return response; + } + + final byte[] body = request.getBody(); + + int queueIdInt = requestHeader.getQueueId(); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + + if (queueIdInt < 0) { + queueIdInt = Math.abs(this.random.nextInt() % 99999999) % topicConfig.getWriteQueueNums(); + } + + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(requestHeader.getTopic()); + msgInner.setQueueId(queueIdInt); + msgInner.setBody(body); + msgInner.setFlag(requestHeader.getFlag()); + MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties())); + msgInner.setPropertiesString(requestHeader.getProperties()); + msgInner.setBornTimestamp(requestHeader.getBornTimestamp()); + msgInner.setBornHost(ctx.channel().remoteAddress()); + msgInner.setStoreHost(this.getStoreHost()); + msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); + + PushReplyResult pushReplyResult = this.pushReplyMessage(ctx, requestHeader, msgInner); + this.handlePushReplyResult(pushReplyResult, response, responseHeader, queueIdInt); + + if (this.brokerController.getBrokerConfig().isStoreReplyMessageEnable()) { + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + this.handlePutMessageResult(putMessageResult, request, msgInner, responseHeader, sendMessageContext, queueIdInt); + } + + return response; + } + + private PushReplyResult pushReplyMessage(final ChannelHandlerContext ctx, + final SendMessageRequestHeader requestHeader, + final Message msg) { + ReplyMessageRequestHeader replyMessageRequestHeader = new ReplyMessageRequestHeader(); + replyMessageRequestHeader.setBornHost(ctx.channel().remoteAddress().toString()); + replyMessageRequestHeader.setStoreHost(this.getStoreHost().toString()); + replyMessageRequestHeader.setStoreTimestamp(System.currentTimeMillis()); + replyMessageRequestHeader.setProducerGroup(requestHeader.getProducerGroup()); + replyMessageRequestHeader.setTopic(requestHeader.getTopic()); + replyMessageRequestHeader.setDefaultTopic(requestHeader.getDefaultTopic()); + replyMessageRequestHeader.setDefaultTopicQueueNums(requestHeader.getDefaultTopicQueueNums()); + replyMessageRequestHeader.setQueueId(requestHeader.getQueueId()); + replyMessageRequestHeader.setSysFlag(requestHeader.getSysFlag()); + replyMessageRequestHeader.setBornTimestamp(requestHeader.getBornTimestamp()); + replyMessageRequestHeader.setFlag(requestHeader.getFlag()); + replyMessageRequestHeader.setProperties(requestHeader.getProperties()); + replyMessageRequestHeader.setReconsumeTimes(requestHeader.getReconsumeTimes()); + replyMessageRequestHeader.setUnitMode(requestHeader.isUnitMode()); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT, replyMessageRequestHeader); + request.setBody(msg.getBody()); + + String senderId = msg.getProperties().get(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT); + PushReplyResult pushReplyResult = new PushReplyResult(false); + + if (senderId != null) { + Channel channel = this.brokerController.getProducerManager().findChannel(senderId); + if (channel != null) { + msg.getProperties().put(MessageConst.PROPERTY_PUSH_REPLY_TIME, String.valueOf(System.currentTimeMillis())); + replyMessageRequestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); + + try { + RemotingCommand pushResponse = this.brokerController.getBroker2Client().callClient(channel, request); + assert pushResponse != null; + switch (pushResponse.getCode()) { + case ResponseCode.SUCCESS: { + pushReplyResult.setPushOk(true); + break; + } + default: { + pushReplyResult.setPushOk(false); + pushReplyResult.setRemark("push reply message to " + senderId + "fail."); + log.warn("push reply message to <{}> return fail, response remark: {}", senderId, pushResponse.getRemark()); + } + } + } catch (RemotingException | InterruptedException e) { + pushReplyResult.setPushOk(false); + pushReplyResult.setRemark("push reply message to " + senderId + "fail."); + log.warn("push reply message to <{}> fail. {}", senderId, channel, e); + } + } else { + pushReplyResult.setPushOk(false); + pushReplyResult.setRemark("push reply message fail, channel of <" + senderId + "> not found."); + log.warn(pushReplyResult.getRemark()); + } + } else { + log.warn(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT + " is null, can not reply message"); + pushReplyResult.setPushOk(false); + pushReplyResult.setRemark("reply message properties[" + MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT + "] is null"); + } + return pushReplyResult; + } + + private void handlePushReplyResult(PushReplyResult pushReplyResult, final RemotingCommand response, + final SendMessageResponseHeader responseHeader, int queueIdInt) { + + if (!pushReplyResult.isPushOk()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(pushReplyResult.getRemark()); + } else { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + //set to zore to avoid client decoding exception + responseHeader.setMsgId("0"); + responseHeader.setQueueId(queueIdInt); + responseHeader.setQueueOffset(0L); + } + } + + private void handlePutMessageResult(PutMessageResult putMessageResult, + final RemotingCommand request, final MessageExt msg, + final SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, + int queueIdInt) { + if (putMessageResult == null) { + log.warn("process reply message, store putMessage return null"); + return; + } + boolean putOk = false; + + switch (putMessageResult.getPutMessageStatus()) { + // Success + case PUT_OK: + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case SLAVE_NOT_AVAILABLE: + putOk = true; + break; + + // Failed + case CREATE_MAPEDFILE_FAILED: + log.info("create mapped file failed, server is busy or broken."); + break; + case MESSAGE_ILLEGAL: + log.info( + "the message is illegal, maybe msg properties length limit 32k."); + break; + case PROPERTIES_SIZE_EXCEEDED: + log.info( + "the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k."); + break; + case SERVICE_NOT_AVAILABLE: + log.info( + "service not available now, maybe disk full, maybe your broker machine memory too small."); + break; + case OS_PAGECACHE_BUSY: + log.info("[PC_SYNCHRONIZED]broker busy, start flow control for a while"); + break; + case UNKNOWN_ERROR: + log.info("UNKNOWN_ERROR"); + break; + default: + log.info("UNKNOWN_ERROR DEFAULT"); + break; + } + + String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER); + if (putOk) { + this.brokerController.getBrokerStatsManager().incTopicPutNums(msg.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); + this.brokerController.getBrokerStatsManager().incTopicPutSize(msg.getTopic(), + putMessageResult.getAppendMessageResult().getWroteBytes()); + this.brokerController.getBrokerStatsManager().incBrokerPutNums(putMessageResult.getAppendMessageResult().getMsgNum()); + + responseHeader.setMsgId(putMessageResult.getAppendMessageResult().getMsgId()); + responseHeader.setQueueId(queueIdInt); + responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset()); + + if (hasSendMessageHook()) { + sendMessageContext.setMsgId(responseHeader.getMsgId()); + sendMessageContext.setQueueId(responseHeader.getQueueId()); + sendMessageContext.setQueueOffset(responseHeader.getQueueOffset()); + + int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount(); + int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes(); + int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount; + + sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS); + sendMessageContext.setCommercialSendTimes(incValue); + sendMessageContext.setCommercialSendSize(wroteSize); + sendMessageContext.setCommercialOwner(owner); + } + } else { + if (hasSendMessageHook()) { + int wroteSize = request.getBody().length; + int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); + + sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE); + sendMessageContext.setCommercialSendTimes(incValue); + sendMessageContext.setCommercialSendSize(wroteSize); + sendMessageContext.setCommercialOwner(owner); + } + } + } + + class PushReplyResult { + boolean pushOk; + String remark; + + public PushReplyResult(boolean pushOk) { + this.pushOk = pushOk; + remark = ""; + } + + public boolean isPushOk() { + return pushOk; + } + + public void setPushOk(boolean pushOk) { + this.pushOk = pushOk; + } + + public String getRemark() { + return remark; + } + + public void setRemark(String remark) { + this.remark = remark; + } + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 8035ae6f185b5c91db73eb0ceea5e0a26f43b3f3..f753ebba4bf4c84f690e06ce0ae5dcc8153c95a8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -343,15 +343,18 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement msgInner.setBody(body); msgInner.setFlag(requestHeader.getFlag()); MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties())); - msgInner.setPropertiesString(requestHeader.getProperties()); msgInner.setBornTimestamp(requestHeader.getBornTimestamp()); msgInner.setBornHost(ctx.channel().remoteAddress()); msgInner.setStoreHost(this.getStoreHost()); msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); + String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName(); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); PutMessageResult putMessageResult = null; Map oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties()); String traFlag = oriProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED); - if (traFlag != null && Boolean.parseBoolean(traFlag)) { + if (traFlag != null && Boolean.parseBoolean(traFlag) + && !(msgInner.getReconsumeTimes() > 0 && msgInner.getDelayTimeLevel() > 0)) { //For client under version 4.6.1 if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) { response.setCode(ResponseCode.NO_PERMISSION); response.setRemark( @@ -536,6 +539,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement messageExtBatch.setBornHost(ctx.channel().remoteAddress()); messageExtBatch.setStoreHost(this.getStoreHost()); messageExtBatch.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); + String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName(); + MessageAccessor.putProperty(messageExtBatch, MessageConst.PROPERTY_CLUSTER, clusterName); PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessages(messageExtBatch); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 8f215cdcb9cde3934b225f1e36e2d638d48dc4fb..199b46d603a72ffb4561d21a6e146df291228390 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -34,11 +34,11 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.sysflag.TopicSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class TopicConfigManager extends ConfigManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -134,6 +134,14 @@ public class TopicConfigManager extends ConfigManager { this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); } } + { + String topic = this.brokerController.getBrokerConfig().getBrokerClusterName() + "_" + MixAll.REPLY_TOPIC_POSTFIX; + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(1); + topicConfig.setWriteQueueNums(1); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } } public boolean isSystemTopic(final String topic) { @@ -144,10 +152,6 @@ public class TopicConfigManager extends ConfigManager { return this.systemTopicList; } - public boolean isTopicCanSendMessage(final String topic) { - return !topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC); - } - public TopicConfig selectTopicConfig(final String topic) { return this.topicConfigTable.get(topic); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicValidator.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicValidator.java new file mode 100644 index 0000000000000000000000000000000000000000..58b1cc86fcdfcdbf38d3c3f9656008fae052b8c6 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicValidator.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.topic; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class TopicValidator { + + private static final String VALID_PATTERN_STR = "^[%|a-zA-Z0-9_-]+$"; + private static final Pattern PATTERN = Pattern.compile(VALID_PATTERN_STR); + private static final int TOPIC_MAX_LENGTH = 127; + + private static boolean regularExpressionMatcher(String origin, Pattern pattern) { + if (pattern == null) { + return true; + } + Matcher matcher = pattern.matcher(origin); + return matcher.matches(); + } + + public static boolean validateTopic(String topic, RemotingCommand response) { + + if (UtilAll.isBlank(topic)) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The specified topic is blank."); + return false; + } + + if (!regularExpressionMatcher(topic, PATTERN)) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The specified topic contains illegal characters, allowing only " + VALID_PATTERN_STR); + return false; + } + + if (topic.length() > TOPIC_MAX_LENGTH) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The specified topic is longer than topic max length."); + return false; + } + + //whether the same with system reserved keyword + if (topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The specified topic is conflict with AUTO_CREATE_TOPIC_KEY_TOPIC."); + return false; + } + + return true; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java index 62507cdfdba6ba5c75dd5001d00b8b68f2b9e926..35d811207ba6687949877daa251730c3f26c77f6 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java @@ -30,6 +30,7 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy; import java.util.concurrent.TimeUnit; public abstract class AbstractTransactionalMessageCheckListener { @@ -48,7 +49,7 @@ public abstract class AbstractTransactionalMessageCheckListener { thread.setName("Transaction-msg-check-thread"); return thread; } - }); + }, new CallerRunsPolicy()); public AbstractTransactionalMessageCheckListener() { } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java index 84a62761bd61a084723abeaf37fedeeb5213735c..67f7a5f5628a0030dc2ffe733e84821eb03f449d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java @@ -131,6 +131,9 @@ public class TransactionalMessageBridge { this.brokerController.getBrokerStatsManager().incGroupGetSize(group, topic, getMessageResult.getBufferTotalSize()); this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount()); + if (foundList == null || foundList.size() == 0) { + break; + } this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, this.brokerController.getMessageStore().now() - foundList.get(foundList.size() - 1) .getStoreTimestamp()); @@ -141,6 +144,7 @@ public class TransactionalMessageBridge { getMessageResult.getStatus(), topic, group, offset); break; case NO_MESSAGE_IN_QUEUE: + case OFFSET_OVERFLOW_ONE: pullStatus = PullStatus.NO_NEW_MSG; LOGGER.warn("No new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", getMessageResult.getStatus(), topic, group, offset); @@ -149,7 +153,6 @@ public class TransactionalMessageBridge { case NO_MATCHED_LOGIC_QUEUE: case OFFSET_FOUND_NULL: case OFFSET_OVERFLOW_BADLY: - case OFFSET_OVERFLOW_ONE: case OFFSET_TOO_SMALL: pullStatus = PullStatus.OFFSET_ILLEGAL; LOGGER.warn("Offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", @@ -175,8 +178,10 @@ public class TransactionalMessageBridge { try { List messageBufferList = getMessageResult.getMessageBufferList(); for (ByteBuffer bb : messageBufferList) { - MessageExt msgExt = MessageDecoder.decode(bb); - foundList.add(msgExt); + MessageExt msgExt = MessageDecoder.decode(bb, true, false); + if (msgExt != null) { + foundList.add(msgExt); + } } } finally { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index e1549b15177e0a2de5fb4160d31d13aee9798f0e..71b575eec098dfd2a2f7ada3b3749ba896700f15 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -159,7 +159,8 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ } if (removeMap.containsKey(i)) { log.info("Half offset {} has been committed/rolled back", i); - removeMap.remove(i); + Long removedOpOffset = removeMap.remove(i); + doneOpOffset.add(removedOpOffset); } else { GetResult getResult = getHalfMsg(messageQueue, i); MessageExt msgExt = getResult.getMsg(); @@ -223,7 +224,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ listener.resolveHalfMsg(msgExt); } else { pullResult = fillOpRemoveMap(removeMap, opQueue, pullResult.getNextBeginOffset(), halfOffset, doneOpOffset); - log.info("The miss offset:{} in messageQueue:{} need to get more opMsg, result is:{}", i, + log.debug("The miss offset:{} in messageQueue:{} need to get more opMsg, result is:{}", i, messageQueue, pullResult); continue; } @@ -292,7 +293,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ } for (MessageExt opMessageExt : opMsg) { Long queueOffset = getLong(new String(opMessageExt.getBody(), TransactionalMessageUtil.charset)); - log.info("Topic: {} tags: {}, OpOffset: {}, HalfOffset: {}", opMessageExt.getTopic(), + log.debug("Topic: {} tags: {}, OpOffset: {}, HalfOffset: {}", opMessageExt.getTopic(), opMessageExt.getTags(), opMessageExt.getQueueOffset(), queueOffset); if (TransactionalMessageUtil.REMOVETAG.equals(opMessageExt.getTags())) { if (queueOffset < miniOffset) { @@ -460,7 +461,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ @Override public boolean deletePrepareMessage(MessageExt msgExt) { if (this.transactionalMessageBridge.putOpMessage(msgExt, TransactionalMessageUtil.REMOVETAG)) { - log.info("Transaction op message write successfully. messageId={}, queueId={} msgExt:{}", msgExt.getMsgId(), msgExt.getQueueId(), msgExt); + log.debug("Transaction op message write successfully. messageId={}, queueId={} msgExt:{}", msgExt.getMsgId(), msgExt.getQueueId(), msgExt); return true; } else { log.error("Transaction op message write failed. messageId is {}, queueId is {}", msgExt.getMsgId(), msgExt.getQueueId()); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java index 08dbb9c754c51c198285e749adcbc2eeb25df0c0..4791ab1f053be7fe648309e8cc0d84c35dcc0c82 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java @@ -19,7 +19,9 @@ package org.apache.rocketmq.broker.client; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import java.lang.reflect.Field; -import java.util.HashMap; +import java.util.Map; + +import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,14 +44,14 @@ public class ProducerManagerTest { @Before public void init() { producerManager = new ProducerManager(); - clientInfo = new ClientChannelInfo(channel); + clientInfo = new ClientChannelInfo(channel, "clientId", LanguageCode.JAVA, 0); } @Test public void scanNotActiveChannel() throws Exception { producerManager.registerProducer(group, clientInfo); assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull(); - + assertThat(producerManager.findChannel("clientId")).isNotNull(); Field field = ProducerManager.class.getDeclaredField("CHANNEL_EXPIRED_TIMEOUT"); field.setAccessible(true); long CHANNEL_EXPIRED_TIMEOUT = field.getLong(producerManager); @@ -57,34 +59,72 @@ public class ProducerManagerTest { when(channel.close()).thenReturn(mock(ChannelFuture.class)); producerManager.scanNotActiveChannel(); assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNull(); + assertThat(producerManager.findChannel("clientId")).isNull(); } @Test public void doChannelCloseEvent() throws Exception { producerManager.registerProducer(group, clientInfo); assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull(); + assertThat(producerManager.findChannel("clientId")).isNotNull(); producerManager.doChannelCloseEvent("127.0.0.1", channel); assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNull(); + assertThat(producerManager.findChannel("clientId")).isNull(); } @Test public void testRegisterProducer() throws Exception { producerManager.registerProducer(group, clientInfo); - HashMap channelMap = producerManager.getGroupChannelTable().get(group); + Map channelMap = producerManager.getGroupChannelTable().get(group); + Channel channel1 = producerManager.findChannel("clientId"); assertThat(channelMap).isNotNull(); + assertThat(channel1).isNotNull(); assertThat(channelMap.get(channel)).isEqualTo(clientInfo); + assertThat(channel1).isEqualTo(channel); } @Test public void unregisterProducer() throws Exception { producerManager.registerProducer(group, clientInfo); - HashMap channelMap = producerManager.getGroupChannelTable().get(group); + Map channelMap = producerManager.getGroupChannelTable().get(group); assertThat(channelMap).isNotNull(); assertThat(channelMap.get(channel)).isEqualTo(clientInfo); - + Channel channel1 = producerManager.findChannel("clientId"); + assertThat(channel1).isNotNull(); + assertThat(channel1).isEqualTo(channel); producerManager.unregisterProducer(group, clientInfo); channelMap = producerManager.getGroupChannelTable().get(group); + channel1 = producerManager.findChannel("clientId"); assertThat(channelMap).isNull(); + assertThat(channel1).isNull(); + + } + + @Test + public void testGetGroupChannelTable() throws Exception { + producerManager.registerProducer(group, clientInfo); + Map oldMap = producerManager.getGroupChannelTable().get(group); + + producerManager.unregisterProducer(group, clientInfo); + assertThat(oldMap.size()).isEqualTo(0); + } + + @Test + public void testGetAvaliableChannel() { + producerManager.registerProducer(group, clientInfo); + + when(channel.isActive()).thenReturn(true); + when(channel.isWritable()).thenReturn(true); + Channel c = producerManager.getAvaliableChannel(group); + assertThat(c).isSameAs(channel); + + when(channel.isWritable()).thenReturn(false); + c = producerManager.getAvaliableChannel(group); + assertThat(c).isSameAs(channel); + + when(channel.isActive()).thenReturn(false); + c = producerManager.getAvaliableChannel(group); + assertThat(c).isNull(); } } \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ClientManageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ClientManageProcessorTest.java index 147c7323803d65a69f535a5175be12ac71e8cf73..3d893ac18657f167f4788a7e8ed5144f50133181 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/ClientManageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ClientManageProcessorTest.java @@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.processor; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import java.util.HashMap; +import java.util.Map; import java.util.UUID; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; @@ -81,7 +82,7 @@ public class ClientManageProcessorTest { @Test public void processRequest_UnRegisterProducer() throws Exception { brokerController.getProducerManager().registerProducer(group, clientChannelInfo); - HashMap channelMap = brokerController.getProducerManager().getGroupChannelTable().get(group); + Map channelMap = brokerController.getProducerManager().getGroupChannelTable().get(group); assertThat(channelMap).isNotNull(); assertThat(channelMap.get(channel)).isEqualTo(clientChannelInfo); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..85c775040f53ef6093e979c8c6adc53a3a36161f --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.HashMap; +import java.util.Map; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.net.Broker2Client; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ReplyMessageProcessorTest { + private ReplyMessageProcessor replyMessageProcessor; + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + @Mock + private ChannelHandlerContext handlerContext; + @Mock + private MessageStore messageStore; + @Mock + private Channel channel; + + private String topic = "FooBar"; + private String group = "FooBarGroup"; + private ClientChannelInfo clientInfo; + @Mock + private Broker2Client broker2Client; + + @Before + public void init() throws IllegalAccessException, NoSuchFieldException { + clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0); + brokerController.setMessageStore(messageStore); + Field field = BrokerController.class.getDeclaredField("broker2Client"); + field.setAccessible(true); + field.set(brokerController, broker2Client); + when(messageStore.now()).thenReturn(System.currentTimeMillis()); + Channel mockChannel = mock(Channel.class); + when(mockChannel.remoteAddress()).thenReturn(new InetSocketAddress(1024)); + when(handlerContext.channel()).thenReturn(mockChannel); + replyMessageProcessor = new ReplyMessageProcessor(brokerController); + } + + @Test + public void testProcessRequest_Success() throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException { + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + brokerController.getProducerManager().registerProducer(group, clientInfo); + final RemotingCommand request = createSendMessageRequestHeaderCommand(RequestCode.SEND_REPLY_MESSAGE); + when(brokerController.getBroker2Client().callClient(any(Channel.class), any(RemotingCommand.class))).thenReturn(createResponse(ResponseCode.SUCCESS, request)); + RemotingCommand responseToReturn = replyMessageProcessor.processRequest(handlerContext, request); + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque()); + } + + private RemotingCommand createSendMessageRequestHeaderCommand(int requestCode) { + SendMessageRequestHeader requestHeader = createSendMessageRequestHeader(); + RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader); + request.setBody(new byte[] {'a'}); + request.makeCustomHeaderToNet(); + return request; + } + + private SendMessageRequestHeader createSendMessageRequestHeader() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setProducerGroup(group); + requestHeader.setTopic(topic); + requestHeader.setDefaultTopic(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC); + requestHeader.setDefaultTopicQueueNums(3); + requestHeader.setQueueId(1); + requestHeader.setSysFlag(0); + requestHeader.setBornTimestamp(System.currentTimeMillis()); + requestHeader.setFlag(124); + requestHeader.setReconsumeTimes(0); + Map map = new HashMap(); + map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, "127.0.0.1"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + return requestHeader; + } + + private RemotingCommand createResponse(int code, RemotingCommand request) { + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + response.setCode(code); + response.setOpaque(request.getOpaque()); + return response; + } +} \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicValidatorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicValidatorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..78be63fc56aaf57c870c44cb095272604e651a78 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicValidatorTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.topic; + +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TopicValidatorTest { + + @Test + public void testTopicValidator_NotPass() { + RemotingCommand response = RemotingCommand.createResponseCommand(-1, ""); + + Boolean res = TopicValidator.validateTopic("", response); + assertThat(res).isFalse(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).contains("The specified topic is blank"); + + clearResponse(response); + res = TopicValidator.validateTopic("../TopicTest", response); + assertThat(res).isFalse(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).contains("The specified topic contains illegal characters"); + + clearResponse(response); + res = TopicValidator.validateTopic(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC, response); + assertThat(res).isFalse(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).contains("The specified topic is conflict with AUTO_CREATE_TOPIC_KEY_TOPIC."); + + clearResponse(response); + res = TopicValidator.validateTopic(generateString(128), response); + assertThat(res).isFalse(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).contains("The specified topic is longer than topic max length."); + + } + + @Test + public void testTopicValidator_Pass() { + RemotingCommand response = RemotingCommand.createResponseCommand(-1, ""); + + Boolean res = TopicValidator.validateTopic("TestTopic", response); + assertThat(res).isTrue(); + assertThat(response.getCode()).isEqualTo(-1); + assertThat(response.getRemark()).isEmpty(); + } + + private static void clearResponse(RemotingCommand response) { + response.setCode(-1); + response.setRemark(""); + } + + private static String generateString(int length) { + StringBuilder stringBuffer = new StringBuilder(); + String tmpStr = "0123456789"; + for (int i = 0; i < length; i++) { + stringBuffer.append(tmpStr); + } + return stringBuffer.toString(); + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java index b1c669c9933b1a0f2af5e0cee918c1eefb8a1717..ebe887285f0ff63fd81c741523a6f1820017fccc 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java @@ -167,6 +167,24 @@ public class TransactionalMessageBridgeTest { assertThat(messageExt).isNotNull(); } + @Test + public void testGetHalfMessageStatusFound() { + when(messageStore + .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) + .thenReturn(createGetMessageResult(GetMessageStatus.FOUND)); + PullResult result = transactionBridge.getHalfMessage(0, 0, 1); + assertThat(result.getPullStatus()).isEqualTo(PullStatus.FOUND); + } + + @Test + public void testGetHalfMessageNull() { + when(messageStore + .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) + .thenReturn(null); + PullResult result = transactionBridge.getHalfMessage(0, 0, 1); + assertThat(result).isNull(); + } + private GetMessageResult createGetMessageResult(GetMessageStatus status) { GetMessageResult getMessageResult = new GetMessageResult(); getMessageResult.setStatus(status); diff --git a/client/pom.xml b/client/pom.xml index db40c5c472b9e867f04b8379a48f8f4babd6becb..95f1cbdac9f8d1f2ea85041645897029330c7036 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java index 87c01a5b3055470be8d1772457afde58d9e55a95..d0ae5e1b8315370c3ce3373c330e9a7d0df2ca84 100644 --- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java @@ -16,7 +16,9 @@ */ package org.apache.rocketmq.client; +import java.util.Collection; import java.util.HashSet; +import java.util.Iterator; import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.UtilAll; @@ -51,6 +53,7 @@ public class ClientConfig { * Offset persistent interval for consumer */ private int persistConsumerOffsetInterval = 1000 * 5; + private long pullTimeDelayMillsWhenException = 1000; private boolean unitMode = false; private String unitName; private boolean vipChannelEnabled = Boolean.parseBoolean(System.getProperty(SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY, "false")); @@ -95,7 +98,6 @@ public class ClientConfig { } } - public String withNamespace(String resource) { return NamespaceUtil.wrapNamespace(this.getNamespace(), resource); } @@ -124,9 +126,21 @@ public class ClientConfig { if (StringUtils.isEmpty(this.getNamespace())) { return queue; } - return new MessageQueue(withNamespace(queue.getTopic()), queue.getBrokerName(), queue.getQueueId()); } + + public Collection queuesWithNamespace(Collection queues) { + if (StringUtils.isEmpty(this.getNamespace())) { + return queues; + } + Iterator iter = queues.iterator(); + while (iter.hasNext()) { + MessageQueue queue = iter.next(); + queue.setTopic(withNamespace(queue.getTopic())); + } + return queues; + } + public void resetClientConfig(final ClientConfig cc) { this.namesrvAddr = cc.namesrvAddr; this.clientIP = cc.clientIP; @@ -135,6 +149,7 @@ public class ClientConfig { this.pollNameServerInterval = cc.pollNameServerInterval; this.heartbeatBrokerInterval = cc.heartbeatBrokerInterval; this.persistConsumerOffsetInterval = cc.persistConsumerOffsetInterval; + this.pullTimeDelayMillsWhenException = cc.pullTimeDelayMillsWhenException; this.unitMode = cc.unitMode; this.unitName = cc.unitName; this.vipChannelEnabled = cc.vipChannelEnabled; @@ -152,6 +167,7 @@ public class ClientConfig { cc.pollNameServerInterval = pollNameServerInterval; cc.heartbeatBrokerInterval = heartbeatBrokerInterval; cc.persistConsumerOffsetInterval = persistConsumerOffsetInterval; + cc.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException; cc.unitMode = unitMode; cc.unitName = unitName; cc.vipChannelEnabled = vipChannelEnabled; @@ -170,6 +186,7 @@ public class ClientConfig { /** * Domain name mode access way does not support the delimiter(;), and only one domain name can be set. + * * @param namesrvAddr name server address */ public void setNamesrvAddr(String namesrvAddr) { @@ -208,6 +225,14 @@ public class ClientConfig { this.persistConsumerOffsetInterval = persistConsumerOffsetInterval; } + public long getPullTimeDelayMillsWhenException() { + return pullTimeDelayMillsWhenException; + } + + public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) { + this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException; + } + public String getUnitName() { return unitName; } @@ -273,12 +298,13 @@ public class ClientConfig { this.accessChannel = accessChannel; } + @Override public String toString() { return "ClientConfig [namesrvAddr=" + namesrvAddr + ", clientIP=" + clientIP + ", instanceName=" + instanceName + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + ", pollNameServerInterval=" + pollNameServerInterval - + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" - + persistConsumerOffsetInterval + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled=" + + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval + + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled=" + vipChannelEnabled + ", useTLS=" + useTLS + ", language=" + language.name() + ", namespace=" + namespace + "]"; } } diff --git a/client/src/main/java/org/apache/rocketmq/client/Validators.java b/client/src/main/java/org/apache/rocketmq/client/Validators.java index 5d6acc0f872bf645ecc28cc1665ab94c8c34725d..d77faf3c222dbcc828c3182ceb4301ba25072d10 100644 --- a/client/src/main/java/org/apache/rocketmq/client/Validators.java +++ b/client/src/main/java/org/apache/rocketmq/client/Validators.java @@ -33,6 +33,7 @@ public class Validators { public static final String VALID_PATTERN_STR = "^[%|a-zA-Z0-9_-]+$"; public static final Pattern PATTERN = Pattern.compile(VALID_PATTERN_STR); public static final int CHARACTER_MAX_LENGTH = 255; + public static final int TOPIC_MAX_LENGTH = 127; /** * @return The resulting {@code String} @@ -53,14 +54,17 @@ public class Validators { if (UtilAll.isBlank(group)) { throw new MQClientException("the specified group is blank", null); } + + if (group.length() > CHARACTER_MAX_LENGTH) { + throw new MQClientException("the specified group is longer than group max length 255.", null); + } + if (!regularExpressionMatcher(group, PATTERN)) { throw new MQClientException(String.format( "the specified group[%s] contains illegal characters, allowing only %s", group, VALID_PATTERN_STR), null); } - if (group.length() > CHARACTER_MAX_LENGTH) { - throw new MQClientException("the specified group is longer than group max length 255.", null); - } + } /** @@ -74,9 +78,6 @@ public class Validators { return matcher.matches(); } - /** - * Validate message - */ public static void checkMessage(Message msg, DefaultMQProducer defaultMQProducer) throws MQClientException { if (null == msg) { @@ -100,9 +101,6 @@ public class Validators { } } - /** - * Validate topic - */ public static void checkTopic(String topic) throws MQClientException { if (UtilAll.isBlank(topic)) { throw new MQClientException("The specified topic is blank", null); @@ -114,8 +112,9 @@ public class Validators { VALID_PATTERN_STR), null); } - if (topic.length() > CHARACTER_MAX_LENGTH) { - throw new MQClientException("The specified topic is longer than topic max length 255.", null); + if (topic.length() > TOPIC_MAX_LENGTH) { + throw new MQClientException( + String.format("The specified topic is longer than topic max length %d.", TOPIC_MAX_LENGTH), null); } //whether the same with system reserved keyword @@ -124,4 +123,5 @@ public class Validators { String.format("The topic[%s] is conflict with AUTO_CREATE_TOPIC_KEY_TOPIC.", topic), null); } } + } diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java b/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java index 62a95dfa5a1ee3acc03bc375dec68fbb867c6751..bc03b14df247f79c5e2e5d07136e77935288c9d3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java +++ b/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java @@ -23,4 +23,6 @@ public class ClientErrorCode { public static final int BROKER_NOT_EXIST_EXCEPTION = 10003; public static final int NO_NAME_SERVER_EXCEPTION = 10004; public static final int NOT_FOUND_TOPIC_EXCEPTION = 10005; + public static final int REQUEST_TIMEOUT_EXCEPTION = 10006; + public static final int CREATE_REPLY_MESSAGE_EXCEPTION = 10007; } \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java new file mode 100644 index 0000000000000000000000000000000000000000..bfef761846ca41271abac0da7da767756fc6d279 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -0,0 +1,465 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.consumer; + +import java.util.Collection; +import java.util.List; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.NamespaceUtil; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.remoting.RPCHook; + +public class DefaultLitePullConsumer extends ClientConfig implements LitePullConsumer { + + private final DefaultLitePullConsumerImpl defaultLitePullConsumerImpl; + + /** + * Consumers belonging to the same consumer group share a group id. The consumers in a group then divides the topic + * as fairly amongst themselves as possible by establishing that each queue is only consumed by a single consumer + * from the group. If all consumers are from the same group, it functions as a traditional message queue. Each + * message would be consumed by one consumer of the group only. When multiple consumer groups exist, the flow of the + * data consumption model aligns with the traditional publish-subscribe model. The messages are broadcast to all + * consumer groups. + */ + private String consumerGroup; + + /** + * Long polling mode, the Consumer connection max suspend time, it is not recommended to modify + */ + private long brokerSuspendMaxTimeMillis = 1000 * 20; + + /** + * Long polling mode, the Consumer connection timeout(must greater than brokerSuspendMaxTimeMillis), it is not + * recommended to modify + */ + private long consumerTimeoutMillisWhenSuspend = 1000 * 30; + + /** + * The socket timeout in milliseconds + */ + private long consumerPullTimeoutMillis = 1000 * 10; + + /** + * Consumption pattern,default is clustering + */ + private MessageModel messageModel = MessageModel.CLUSTERING; + /** + * Message queue listener + */ + private MessageQueueListener messageQueueListener; + /** + * Offset Storage + */ + private OffsetStore offsetStore; + + /** + * Queue allocation algorithm + */ + private AllocateMessageQueueStrategy allocateMessageQueueStrategy = new AllocateMessageQueueAveragely(); + /** + * Whether the unit of subscription group + */ + private boolean unitMode = false; + + /** + * The flag for auto commit offset + */ + private boolean autoCommit = true; + + /** + * Pull thread number + */ + private int pullThreadNums = 20; + + /** + * Maximum commit offset interval time in milliseconds. + */ + private long autoCommitIntervalMillis = 5 * 1000; + + /** + * Maximum number of messages pulled each time. + */ + private int pullBatchSize = 10; + + /** + * Flow control threshold for consume request, each consumer will cache at most 10000 consume requests by default. + * Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit + */ + private long pullThresholdForAll = 10000; + + /** + * Consume max span offset. + */ + private int consumeMaxSpan = 2000; + + /** + * Flow control threshold on queue level, each message queue will cache at most 1000 messages by default, Consider + * the {@code pullBatchSize}, the instantaneous value may exceed the limit + */ + private int pullThresholdForQueue = 1000; + + /** + * Limit the cached message size on queue level, each message queue will cache at most 100 MiB messages by default, + * Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit + * + *

+ * The size of a message only measured by message body, so it's not accurate + */ + private int pullThresholdSizeForQueue = 100; + + /** + * The poll timeout in milliseconds + */ + private long pollTimeoutMillis = 1000 * 5; + + /** + * Interval time in in milliseconds for checking changes in topic metadata. + */ + private long topicMetadataCheckIntervalMillis = 30 * 1000; + + private ConsumeFromWhere consumeFromWhere = ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET; + + /** + * Backtracking consumption time with second precision. Time format is 20131223171201
Implying Seventeen twelve + * and 01 seconds on December 23, 2013 year
Default backtracking consumption time Half an hour ago. + */ + private String consumeTimestamp = UtilAll.timeMillisToHumanString3(System.currentTimeMillis() - (1000 * 60 * 30)); + + /** + * Default constructor. + */ + public DefaultLitePullConsumer() { + this(null, MixAll.DEFAULT_CONSUMER_GROUP, null); + } + + /** + * Constructor specifying consumer group. + * + * @param consumerGroup Consumer group. + */ + public DefaultLitePullConsumer(final String consumerGroup) { + this(null, consumerGroup, null); + } + + /** + * Constructor specifying RPC hook. + * + * @param rpcHook RPC hook to execute before each remoting command. + */ + public DefaultLitePullConsumer(RPCHook rpcHook) { + this(null, MixAll.DEFAULT_CONSUMER_GROUP, rpcHook); + } + + /** + * Constructor specifying consumer group, RPC hook + * + * @param consumerGroup Consumer group. + * @param rpcHook RPC hook to execute before each remoting command. + */ + public DefaultLitePullConsumer(final String consumerGroup, RPCHook rpcHook) { + this(null, consumerGroup, rpcHook); + } + + /** + * Constructor specifying namespace, consumer group and RPC hook. + * + * @param consumerGroup Consumer group. + * @param rpcHook RPC hook to execute before each remoting command. + */ + public DefaultLitePullConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook) { + this.namespace = namespace; + this.consumerGroup = consumerGroup; + defaultLitePullConsumerImpl = new DefaultLitePullConsumerImpl(this, rpcHook); + } + + @Override + public void start() throws MQClientException { + setConsumerGroup(NamespaceUtil.wrapNamespace(this.getNamespace(), this.consumerGroup)); + this.defaultLitePullConsumerImpl.start(); + } + + @Override + public void shutdown() { + this.defaultLitePullConsumerImpl.shutdown(); + } + + @Override + public void subscribe(String topic, String subExpression) throws MQClientException { + this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), subExpression); + } + + @Override + public void subscribe(String topic, MessageSelector messageSelector) throws MQClientException { + this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), messageSelector); + } + + @Override + public void unsubscribe(String topic) { + this.defaultLitePullConsumerImpl.unsubscribe(withNamespace(topic)); + } + + @Override + public void assign(Collection messageQueues) { + defaultLitePullConsumerImpl.assign(queuesWithNamespace(messageQueues)); + } + + @Override + public List poll() { + return defaultLitePullConsumerImpl.poll(this.getPollTimeoutMillis()); + } + + @Override + public List poll(long timeout) { + return defaultLitePullConsumerImpl.poll(timeout); + } + + @Override + public void seek(MessageQueue messageQueue, long offset) throws MQClientException { + this.defaultLitePullConsumerImpl.seek(queueWithNamespace(messageQueue), offset); + } + + @Override + public void pause(Collection messageQueues) { + this.defaultLitePullConsumerImpl.pause(queuesWithNamespace(messageQueues)); + } + + @Override + public void resume(Collection messageQueues) { + this.defaultLitePullConsumerImpl.resume(queuesWithNamespace(messageQueues)); + } + + @Override + public Collection fetchMessageQueues(String topic) throws MQClientException { + return this.defaultLitePullConsumerImpl.fetchMessageQueues(withNamespace(topic)); + } + + @Override + public Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException { + return this.defaultLitePullConsumerImpl.searchOffset(queueWithNamespace(messageQueue), timestamp); + } + + @Override + public void registerTopicMessageQueueChangeListener(String topic, + TopicMessageQueueChangeListener topicMessageQueueChangeListener) throws MQClientException { + this.defaultLitePullConsumerImpl.registerTopicMessageQueueChangeListener(withNamespace(topic), topicMessageQueueChangeListener); + } + + @Override + public void commitSync() { + this.defaultLitePullConsumerImpl.commitAll(); + } + + @Override + public Long committed(MessageQueue messageQueue) throws MQClientException { + return this.defaultLitePullConsumerImpl.committed(messageQueue); + } + + @Override + public void updateNameServerAddress(String nameServerAddress) { + this.defaultLitePullConsumerImpl.updateNameServerAddr(nameServerAddress); + } + + @Override + public void seekToBegin(MessageQueue messageQueue) throws MQClientException { + this.defaultLitePullConsumerImpl.seekToBegin(messageQueue); + } + + @Override + public void seekToEnd(MessageQueue messageQueue) throws MQClientException { + this.defaultLitePullConsumerImpl.seekToEnd(messageQueue); + } + + @Override + public boolean isAutoCommit() { + return autoCommit; + } + + @Override + public void setAutoCommit(boolean autoCommit) { + this.autoCommit = autoCommit; + } + + public int getPullThreadNums() { + return pullThreadNums; + } + + public void setPullThreadNums(int pullThreadNums) { + this.pullThreadNums = pullThreadNums; + } + + public long getAutoCommitIntervalMillis() { + return autoCommitIntervalMillis; + } + + public void setAutoCommitIntervalMillis(long autoCommitIntervalMillis) { + this.autoCommitIntervalMillis = autoCommitIntervalMillis; + } + + public int getPullBatchSize() { + return pullBatchSize; + } + + public void setPullBatchSize(int pullBatchSize) { + this.pullBatchSize = pullBatchSize; + } + + public long getPullThresholdForAll() { + return pullThresholdForAll; + } + + public void setPullThresholdForAll(long pullThresholdForAll) { + this.pullThresholdForAll = pullThresholdForAll; + } + + public int getConsumeMaxSpan() { + return consumeMaxSpan; + } + + public void setConsumeMaxSpan(int consumeMaxSpan) { + this.consumeMaxSpan = consumeMaxSpan; + } + + public int getPullThresholdForQueue() { + return pullThresholdForQueue; + } + + public void setPullThresholdForQueue(int pullThresholdForQueue) { + this.pullThresholdForQueue = pullThresholdForQueue; + } + + public int getPullThresholdSizeForQueue() { + return pullThresholdSizeForQueue; + } + + public void setPullThresholdSizeForQueue(int pullThresholdSizeForQueue) { + this.pullThresholdSizeForQueue = pullThresholdSizeForQueue; + } + + public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() { + return allocateMessageQueueStrategy; + } + + public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + } + + public long getBrokerSuspendMaxTimeMillis() { + return brokerSuspendMaxTimeMillis; + } + + public long getPollTimeoutMillis() { + return pollTimeoutMillis; + } + + public void setPollTimeoutMillis(long pollTimeoutMillis) { + this.pollTimeoutMillis = pollTimeoutMillis; + } + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + public void setOffsetStore(OffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + public boolean isUnitMode() { + return unitMode; + } + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + public MessageModel getMessageModel() { + return messageModel; + } + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public MessageQueueListener getMessageQueueListener() { + return messageQueueListener; + } + + public void setMessageQueueListener(MessageQueueListener messageQueueListener) { + this.messageQueueListener = messageQueueListener; + } + + public long getConsumerPullTimeoutMillis() { + return consumerPullTimeoutMillis; + } + + public void setConsumerPullTimeoutMillis(long consumerPullTimeoutMillis) { + this.consumerPullTimeoutMillis = consumerPullTimeoutMillis; + } + + public long getConsumerTimeoutMillisWhenSuspend() { + return consumerTimeoutMillisWhenSuspend; + } + + public void setConsumerTimeoutMillisWhenSuspend(long consumerTimeoutMillisWhenSuspend) { + this.consumerTimeoutMillisWhenSuspend = consumerTimeoutMillisWhenSuspend; + } + + public long getTopicMetadataCheckIntervalMillis() { + return topicMetadataCheckIntervalMillis; + } + + public void setTopicMetadataCheckIntervalMillis(long topicMetadataCheckIntervalMillis) { + this.topicMetadataCheckIntervalMillis = topicMetadataCheckIntervalMillis; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public ConsumeFromWhere getConsumeFromWhere() { + return consumeFromWhere; + } + + public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) { + if (consumeFromWhere != ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET + && consumeFromWhere != ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET + && consumeFromWhere != ConsumeFromWhere.CONSUME_FROM_TIMESTAMP) { + throw new RuntimeException("Invalid ConsumeFromWhere Value", null); + } + this.consumeFromWhere = consumeFromWhere; + } + + public String getConsumeTimestamp() { + return consumeTimestamp; + } + + public void setConsumeTimestamp(String consumeTimestamp) { + this.consumeTimestamp = consumeTimestamp; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index f3b6caaa71f23861f8bc767e5f64178cef1ad752..0876a94e4c536b9691840373c5daa2a2c6aa9463 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -35,9 +35,13 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; /** - * Default pulling consumer + * Default pulling consumer. + * This class will be removed in 2022, and a better implementation {@link DefaultLitePullConsumer} is recommend to use + * in the scenario of actively pulling messages. */ +@Deprecated public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsumer { + protected final transient DefaultMQPullConsumerImpl defaultMQPullConsumerImpl; /** diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index 339f799f9ac867688ad30d463a79540c6ac852c0..6ad0fc308ecfbf11951a4f56ac778c1f58a289ec 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -388,7 +388,7 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook); if (enableMsgTrace) { try { - AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(customizedTraceTopic, rpcHook); + AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, customizedTraceTopic, rpcHook); dispatcher.setHostConsumer(this.getDefaultMQPushConsumerImpl()); traceDispatcher = dispatcher; this.getDefaultMQPushConsumerImpl().registerConsumeMessageHook( diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java new file mode 100644 index 0000000000000000000000000000000000000000..ce2228803383d7fc1502b0fa5babd9bd1ec5c0ac --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.consumer; + +import java.util.Collection; +import java.util.List; + +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; + +public interface LitePullConsumer { + + /** + * Start the consumer + */ + void start() throws MQClientException; + + /** + * Shutdown the consumer + */ + void shutdown(); + + /** + * Subscribe some topic with subExpression + * + * @param subExpression subscription expression.it only support or operation such as "tag1 || tag2 || tag3"
if + * null or * expression,meaning subscribe all + * @throws MQClientException if there is any client error. + */ + void subscribe(final String topic, final String subExpression) throws MQClientException; + + /** + * Subscribe some topic with selector. + * + * @param selector message selector({@link MessageSelector}), can be null. + * @throws MQClientException if there is any client error. + */ + void subscribe(final String topic, final MessageSelector selector) throws MQClientException; + + /** + * Unsubscribe consumption some topic + * + * @param topic Message topic that needs to be unsubscribe. + */ + void unsubscribe(final String topic); + + /** + * Manually assign a list of message queues to this consumer. This interface does not allow for incremental + * assignment and will replace the previous assignment (if there is one). + * + * @param messageQueues Message queues that needs to be assigned. + */ + void assign(Collection messageQueues); + + /** + * Fetch data for the topics or partitions specified using assign API + * + * @return list of message, can be null. + */ + List poll(); + + /** + * Fetch data for the topics or partitions specified using assign API + * + * @param timeout The amount time, in milliseconds, spent waiting in poll if data is not available. Must not be + * negative + * @return list of message, can be null. + */ + List poll(long timeout); + + /** + * Overrides the fetch offsets that the consumer will use on the next poll. If this API is invoked for the same + * message queue more than once, the latest offset will be used on the next poll(). Note that you may lose data if + * this API is arbitrarily used in the middle of consumption. + * + * @param messageQueue + * @param offset + */ + void seek(MessageQueue messageQueue, long offset) throws MQClientException; + + /** + * Suspend pulling from the requested message queues. + * + * Because of the implementation of pre-pull, fetch data in {@link #poll()} will not stop immediately until the + * messages of the requested message queues drain. + * + * Note that this method does not affect message queue subscription. In particular, it does not cause a group + * rebalance. + * + * @param messageQueues Message queues that needs to be paused. + */ + void pause(Collection messageQueues); + + /** + * Resume specified message queues which have been paused with {@link #pause(Collection)}. + * + * @param messageQueues Message queues that needs to be resumed. + */ + void resume(Collection messageQueues); + + /** + * Whether to enable auto-commit consume offset. + * + * @return true if enable auto-commit, false if disable auto-commit. + */ + boolean isAutoCommit(); + + /** + * Set whether to enable auto-commit consume offset. + * + * @param autoCommit Whether to enable auto-commit. + */ + void setAutoCommit(boolean autoCommit); + + /** + * Get metadata about the message queues for a given topic. + * + * @param topic The topic that need to get metadata. + * @return collection of message queues + * @throws MQClientException if there is any client error. + */ + Collection fetchMessageQueues(String topic) throws MQClientException; + + /** + * Look up the offsets for the given message queue by timestamp. The returned offset for each message queue is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding message + * queue. + * + * @param messageQueue Message queues that needs to get offset by timestamp. + * @param timestamp + * @return offset + * @throws MQClientException if there is any client error. + */ + Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException; + + /** + * Manually commit consume offset. + */ + void commitSync(); + + /** + * Get the last committed offset for the given message queue. + * + * @param messageQueue + * @return offset, if offset equals -1 means no offset in broker. + * @throws MQClientException if there is any client error. + */ + Long committed(MessageQueue messageQueue) throws MQClientException; + + /** + * Register a callback for sensing topic metadata changes. + * + * @param topic The topic that need to monitor. + * @param topicMessageQueueChangeListener Callback when topic metadata changes, refer {@link + * TopicMessageQueueChangeListener} + * @throws MQClientException if there is any client error. + */ + void registerTopicMessageQueueChangeListener(String topic, + TopicMessageQueueChangeListener topicMessageQueueChangeListener) throws MQClientException; + + /** + * Update name server addresses. + */ + void updateNameServerAddress(String nameServerAddress); + + /** + * Overrides the fetch offsets with the begin offset that the consumer will use on the next poll. If this API is + * invoked for the same message queue more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption. + * + * @param messageQueue + */ + void seekToBegin(MessageQueue messageQueue)throws MQClientException; + + /** + * Overrides the fetch offsets with the end offset that the consumer will use on the next poll. If this API is + * invoked for the same message queue more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption. + * + * @param messageQueue + */ + void seekToEnd(MessageQueue messageQueue)throws MQClientException; +} diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java index 28b807c2ed89ac460d9cfd0e61a35dc144c42242..a8e96283f9d06c11a29197d1f62f0c494abb5d56 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java @@ -169,4 +169,5 @@ public interface MQPullConsumer extends MQConsumer { */ void sendMessageBack(MessageExt msg, int delayLevel, String brokerName, String consumerGroup) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java index 5436688052cc52634782d5f4f344933470159973..4d57313f8fa215b6b0fba0b3239c13b01d7425b3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java @@ -32,7 +32,9 @@ import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; /** - * Schedule service for pull consumer + * Schedule service for pull consumer. + * This Consumer will be removed in 2022, and a better implementation {@link + * DefaultLitePullConsumer} is recommend to use in the scenario of actively pulling messages. */ public class MQPullConsumerScheduleService { private final InternalLogger log = ClientLogger.getLog(); @@ -157,7 +159,7 @@ public class MQPullConsumerScheduleService { } } - class PullTaskImpl implements Runnable { + public class PullTaskImpl implements Runnable { private final MessageQueue messageQueue; private volatile boolean cancelled = false; diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/TopicMessageQueueChangeListener.java b/client/src/main/java/org/apache/rocketmq/client/consumer/TopicMessageQueueChangeListener.java new file mode 100644 index 0000000000000000000000000000000000000000..fa6fd134e230118654eac42d68ee5296305c0e06 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/TopicMessageQueueChangeListener.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.consumer; + +import java.util.Set; +import org.apache.rocketmq.common.message.MessageQueue; + +public interface TopicMessageQueueChangeListener { + /** + * This method will be invoked in the condition of queue numbers changed, These scenarios occur when the topic is + * expanded or shrunk. + * + * @param messageQueues + */ + void onChanged(String topic, Set messageQueues); +} diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java index c1524e10777981e25ef03326ef0abbc8727041e8..6b762383717bd01889b355334540f0fca061d6ec 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java @@ -117,25 +117,24 @@ public class RemoteBrokerOffsetStore implements OffsetStore { return; final HashSet unusedMQ = new HashSet(); - if (!mqs.isEmpty()) { - for (Map.Entry entry : this.offsetTable.entrySet()) { - MessageQueue mq = entry.getKey(); - AtomicLong offset = entry.getValue(); - if (offset != null) { - if (mqs.contains(mq)) { - try { - this.updateConsumeOffsetToBroker(mq, offset.get()); - log.info("[persistAll] Group: {} ClientId: {} updateConsumeOffsetToBroker {} {}", - this.groupName, - this.mQClientFactory.getClientId(), - mq, - offset.get()); - } catch (Exception e) { - log.error("updateConsumeOffsetToBroker exception, " + mq.toString(), e); - } - } else { - unusedMQ.add(mq); + + for (Map.Entry entry : this.offsetTable.entrySet()) { + MessageQueue mq = entry.getKey(); + AtomicLong offset = entry.getValue(); + if (offset != null) { + if (mqs.contains(mq)) { + try { + this.updateConsumeOffsetToBroker(mq, offset.get()); + log.info("[persistAll] Group: {} ClientId: {} updateConsumeOffsetToBroker {} {}", + this.groupName, + this.mQClientFactory.getClientId(), + mq, + offset.get()); + } catch (Exception e) { + log.error("updateConsumeOffsetToBroker exception, " + mq.toString(), e); } + } else { + unusedMQ.add(mq); } } } @@ -187,8 +186,7 @@ public class RemoteBrokerOffsetStore implements OffsetStore { } /** - * Update the Consumer Offset in one way, once the Master is off, updated to Slave, - * here need to be optimized. + * Update the Consumer Offset in one way, once the Master is off, updated to Slave, here need to be optimized. */ private void updateConsumeOffsetToBroker(MessageQueue mq, long offset) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { @@ -196,15 +194,13 @@ public class RemoteBrokerOffsetStore implements OffsetStore { } /** - * Update the Consumer Offset synchronously, once the Master is off, updated to Slave, - * here need to be optimized. + * Update the Consumer Offset synchronously, once the Master is off, updated to Slave, here need to be optimized. */ @Override public void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); if (null == findBrokerResult) { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); } diff --git a/client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java b/client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java new file mode 100644 index 0000000000000000000000000000000000000000..2d756ece617d1da158157ed382bb9ffd799dfc63 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.exception; + +import org.apache.rocketmq.common.UtilAll; + +public class RequestTimeoutException extends Exception { + private static final long serialVersionUID = -5758410930844185841L; + private int responseCode; + private String errorMessage; + + public RequestTimeoutException(String errorMessage, Throwable cause) { + super(errorMessage, cause); + this.responseCode = -1; + this.errorMessage = errorMessage; + } + + public RequestTimeoutException(int responseCode, String errorMessage) { + super("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " + + errorMessage); + this.responseCode = responseCode; + this.errorMessage = errorMessage; + } + + public int getResponseCode() { + return responseCode; + } + + public RequestTimeoutException setResponseCode(final int responseCode) { + this.responseCode = responseCode; + return this; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(final String errorMessage) { + this.errorMessage = errorMessage; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index 0bd810a1e1db1b4df45e091052e2aac7394a2130..5861bc4baa18405eded9e4c4da35ad00eb145643 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -16,16 +16,19 @@ */ package org.apache.rocketmq.client.impl; +import io.netty.channel.ChannelHandlerContext; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; - -import io.netty.channel.ChannelHandlerContext; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.impl.producer.MQProducerInner; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.RequestFutureTable; +import org.apache.rocketmq.client.producer.RequestResponseFuture; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; @@ -42,14 +45,16 @@ import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRe import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; import org.apache.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; import org.apache.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; +import org.apache.rocketmq.common.protocol.header.ReplyMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; - public class ClientRemotingProcessor implements NettyRequestProcessor { private final InternalLogger log = ClientLogger.getLog(); private final MQClientInstance mqClientFactory; @@ -76,6 +81,9 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { case RequestCode.CONSUME_MESSAGE_DIRECTLY: return this.consumeMessageDirectly(ctx, request); + + case RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT: + return this.receiveReplyMessage(ctx, request); default: break; } @@ -213,4 +221,73 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { return response; } + + private RemotingCommand receiveReplyMessage(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + long receiveTime = System.currentTimeMillis(); + ReplyMessageRequestHeader requestHeader = (ReplyMessageRequestHeader) request.decodeCommandCustomHeader(ReplyMessageRequestHeader.class); + + try { + MessageExt msg = new MessageExt(); + msg.setTopic(requestHeader.getTopic()); + msg.setQueueId(requestHeader.getQueueId()); + msg.setStoreTimestamp(requestHeader.getStoreTimestamp()); + + if (requestHeader.getBornHost() != null) { + msg.setBornHost(RemotingUtil.string2SocketAddress(requestHeader.getBornHost())); + } + + if (requestHeader.getStoreHost() != null) { + msg.setStoreHost(RemotingUtil.string2SocketAddress(requestHeader.getStoreHost())); + } + + byte[] body = request.getBody(); + if ((requestHeader.getSysFlag() & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) { + try { + body = UtilAll.uncompress(body); + } catch (IOException e) { + log.warn("err when uncompress constant", e); + } + } + msg.setBody(body); + msg.setFlag(requestHeader.getFlag()); + MessageAccessor.setProperties(msg, MessageDecoder.string2messageProperties(requestHeader.getProperties())); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REPLY_MESSAGE_ARRIVE_TIME, String.valueOf(receiveTime)); + msg.setBornTimestamp(requestHeader.getBornTimestamp()); + msg.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); + log.debug("receive reply message :{}", msg); + + processReplyMessage(msg); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } catch (Exception e) { + log.warn("unknown err when receiveReplyMsg", e); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("process reply message fail"); + } + return response; + } + + private void processReplyMessage(MessageExt replyMsg) { + final String correlationId = replyMsg.getUserProperty(MessageConst.PROPERTY_CORRELATION_ID); + final RequestResponseFuture requestResponseFuture = RequestFutureTable.getRequestFutureTable().get(correlationId); + if (requestResponseFuture != null) { + requestResponseFuture.putResponseMessage(replyMsg); + + RequestFutureTable.getRequestFutureTable().remove(correlationId); + + if (requestResponseFuture.getRequestCallback() != null) { + requestResponseFuture.getRequestCallback().onSuccess(replyMsg); + } else { + requestResponseFuture.putResponseMessage(replyMsg); + } + } else { + String bornHost = replyMsg.getBornHostString(); + log.warn(String.format("receive reply message, but not matched any request, CorrelationId: %s , reply from host: %s", + correlationId, bornHost)); + } + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index ca89d613060b9b1aa4cb818760b4f9767b5f68ef..9dbd55201dc6bea8ff1046e59be1fbd674a2b63b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -28,6 +28,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.rocketmq.client.QueryResult; +import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.factory.MQClientInstance; @@ -80,6 +81,7 @@ public class MQAdminImpl { public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { try { + Validators.checkTopic(newTopic); TopicRouteData topicRouteData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(key, timeoutMillis); List brokerDataList = topicRouteData.getBrokerDatas(); if (brokerDataList != null && !brokerDataList.isEmpty()) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 52c39923b8de470addfa54365bcd73858ea84e19..c7d5a06dfd33abee62a83c535e1460ae02db7511 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.PullCallback; @@ -44,6 +43,7 @@ import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; @@ -96,6 +96,7 @@ import org.apache.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequest import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.GetBrokerAclConfigResponseHeader; +import org.apache.rocketmq.common.protocol.header.GetBrokerClusterAclConfigResponseBody; import org.apache.rocketmq.common.protocol.header.GetConsumeStatsInBrokerHeader; import org.apache.rocketmq.common.protocol.header.GetConsumeStatsRequestHeader; import org.apache.rocketmq.common.protocol.header.GetConsumerConnectionListRequestHeader; @@ -199,6 +200,8 @@ public class MQClientAPIImpl { this.remotingClient.registerProcessor(RequestCode.GET_CONSUMER_RUNNING_INFO, this.clientRemotingProcessor, null); this.remotingClient.registerProcessor(RequestCode.CONSUME_MESSAGE_DIRECTLY, this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT, this.clientRemotingProcessor, null); } public List getNameServerAddressList() { @@ -302,8 +305,8 @@ public class MQClientAPIImpl { requestHeader.setDefaultGroupPerm(plainAccessConfig.getDefaultGroupPerm()); requestHeader.setDefaultTopicPerm(plainAccessConfig.getDefaultTopicPerm()); requestHeader.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress()); - requestHeader.setTopicPerms(UtilAll.List2String(plainAccessConfig.getTopicPerms(), ",")); - requestHeader.setGroupPerms(UtilAll.List2String(plainAccessConfig.getGroupPerms(), ",")); + requestHeader.setTopicPerms(UtilAll.list2String(plainAccessConfig.getTopicPerms(), ",")); + requestHeader.setGroupPerms(UtilAll.list2String(plainAccessConfig.getGroupPerms(), ",")); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_ACL_CONFIG, requestHeader); @@ -391,6 +394,30 @@ public class MQClientAPIImpl { } + public AclConfig getBrokerClusterConfig(final String addr, final long timeoutMillis) throws RemotingCommandException, InterruptedException, RemotingTimeoutException, + RemotingSendRequestException, RemotingConnectException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_ACL_CONFIG, null); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + GetBrokerClusterAclConfigResponseBody body = + GetBrokerClusterAclConfigResponseBody.decode(response.getBody(), GetBrokerClusterAclConfigResponseBody.class); + AclConfig aclConfig = new AclConfig(); + aclConfig.setGlobalWhiteAddrs(body.getGlobalWhiteAddrs()); + aclConfig.setPlainAccessConfigs(body.getPlainAccessConfigs()); + return aclConfig; + } + } + default: + break; + } + throw new MQBrokerException(response.getCode(), response.getRemark()); + + } + public SendResult sendMessage( final String addr, final String brokerName, @@ -420,13 +447,23 @@ public class MQClientAPIImpl { ) throws RemotingException, MQBrokerException, InterruptedException { long beginStartTime = System.currentTimeMillis(); RemotingCommand request = null; - if (sendSmartMsg || msg instanceof MessageBatch) { - SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); - request = RemotingCommand.createRequestCommand(msg instanceof MessageBatch ? RequestCode.SEND_BATCH_MESSAGE : RequestCode.SEND_MESSAGE_V2, requestHeaderV2); + String msgType = msg.getProperty(MessageConst.PROPERTY_MESSAGE_TYPE); + boolean isReply = msgType != null && msgType.equals(MixAll.REPLY_MESSAGE_FLAG); + if (isReply) { + if (sendSmartMsg) { + SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); + request = RemotingCommand.createRequestCommand(RequestCode.SEND_REPLY_MESSAGE_V2, requestHeaderV2); + } else { + request = RemotingCommand.createRequestCommand(RequestCode.SEND_REPLY_MESSAGE, requestHeader); + } } else { - request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader); + if (sendSmartMsg || msg instanceof MessageBatch) { + SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); + request = RemotingCommand.createRequestCommand(msg instanceof MessageBatch ? RequestCode.SEND_BATCH_MESSAGE : RequestCode.SEND_MESSAGE_V2, requestHeaderV2); + } else { + request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader); + } } - request.setBody(msg.getBody()); switch (communicationMode) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java index 31553a67ae5fd28cfc2605ecc633f7aaf5edbeb4..053c049c9cd703832afe9044b92fd8e82fa2b719 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientManager.java @@ -40,11 +40,11 @@ public class MQClientManager { return instance; } - public MQClientInstance getAndCreateMQClientInstance(final ClientConfig clientConfig) { - return getAndCreateMQClientInstance(clientConfig, null); + public MQClientInstance getOrCreateMQClientInstance(final ClientConfig clientConfig) { + return getOrCreateMQClientInstance(clientConfig, null); } - public MQClientInstance getAndCreateMQClientInstance(final ClientConfig clientConfig, RPCHook rpcHook) { + public MQClientInstance getOrCreateMQClientInstance(final ClientConfig clientConfig, RPCHook rpcHook) { String clientId = clientConfig.buildMQClientId(); MQClientInstance instance = this.factoryTable.get(clientId); if (null == instance) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java new file mode 100644 index 0000000000000000000000000000000000000000..fad0b4f1b584d301c7683aac3be36531627e2721 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.common.message.MessageQueue; + +public class AssignedMessageQueue { + + private final ConcurrentHashMap assignedMessageQueueState; + + private RebalanceImpl rebalanceImpl; + + public AssignedMessageQueue() { + assignedMessageQueueState = new ConcurrentHashMap(); + } + + public void setRebalanceImpl(RebalanceImpl rebalanceImpl) { + this.rebalanceImpl = rebalanceImpl; + } + + public Set messageQueues() { + return assignedMessageQueueState.keySet(); + } + + public boolean isPaused(MessageQueue messageQueue) { + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.isPaused(); + } + return true; + } + + public void pause(Collection messageQueues) { + for (MessageQueue messageQueue : messageQueues) { + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (assignedMessageQueueState.get(messageQueue) != null) { + messageQueueState.setPaused(true); + } + } + } + + public void resume(Collection messageQueueCollection) { + for (MessageQueue messageQueue : messageQueueCollection) { + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (assignedMessageQueueState.get(messageQueue) != null) { + messageQueueState.setPaused(false); + } + } + } + + public ProcessQueue getProcessQueue(MessageQueue messageQueue) { + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.getProcessQueue(); + } + return null; + } + + public long getPullOffset(MessageQueue messageQueue) { + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.getPullOffset(); + } + return -1; + } + + public void updatePullOffset(MessageQueue messageQueue, long offset) { + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + messageQueueState.setPullOffset(offset); + } + } + + public long getConsumerOffset(MessageQueue messageQueue) { + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.getConsumeOffset(); + } + return -1; + } + + public void updateConsumeOffset(MessageQueue messageQueue, long offset) { + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + messageQueueState.setConsumeOffset(offset); + } + } + + public void setSeekOffset(MessageQueue messageQueue, long offset) { + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + messageQueueState.setSeekOffset(offset); + } + } + + public long getSeekOffset(MessageQueue messageQueue) { + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.getSeekOffset(); + } + return -1; + } + + public void updateAssignedMessageQueue(String topic, Collection assigned) { + synchronized (this.assignedMessageQueueState) { + Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + if (!assigned.contains(next.getKey())) { + next.getValue().getProcessQueue().setDropped(true); + it.remove(); + } + } + } + addAssignedMessageQueue(assigned); + } + } + + public void updateAssignedMessageQueue(Collection assigned) { + synchronized (this.assignedMessageQueueState) { + Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (!assigned.contains(next.getKey())) { + next.getValue().getProcessQueue().setDropped(true); + it.remove(); + } + } + addAssignedMessageQueue(assigned); + } + } + + private void addAssignedMessageQueue(Collection assigned) { + for (MessageQueue messageQueue : assigned) { + if (!this.assignedMessageQueueState.containsKey(messageQueue)) { + MessageQueueState messageQueueState; + if (rebalanceImpl != null && rebalanceImpl.getProcessQueueTable().get(messageQueue) != null) { + messageQueueState = new MessageQueueState(messageQueue, rebalanceImpl.getProcessQueueTable().get(messageQueue)); + } else { + ProcessQueue processQueue = new ProcessQueue(); + messageQueueState = new MessageQueueState(messageQueue, processQueue); + } + this.assignedMessageQueueState.put(messageQueue, messageQueueState); + } + } + } + + public void removeAssignedMessageQueue(String topic) { + synchronized (this.assignedMessageQueueState) { + Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + it.remove(); + } + } + } + } + + public Set getAssignedMessageQueues() { + return this.assignedMessageQueueState.keySet(); + } + + private class MessageQueueState { + private MessageQueue messageQueue; + private ProcessQueue processQueue; + private volatile boolean paused = false; + private volatile long pullOffset = -1; + private volatile long consumeOffset = -1; + private volatile long seekOffset = -1; + + private MessageQueueState(MessageQueue messageQueue, ProcessQueue processQueue) { + this.messageQueue = messageQueue; + this.processQueue = processQueue; + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + public boolean isPaused() { + return paused; + } + + public void setPaused(boolean paused) { + this.paused = paused; + } + + public long getPullOffset() { + return pullOffset; + } + + public void setPullOffset(long pullOffset) { + this.pullOffset = pullOffset; + } + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + public void setProcessQueue(ProcessQueue processQueue) { + this.processQueue = processQueue; + } + + public long getConsumeOffset() { + return consumeOffset; + } + + public void setConsumeOffset(long consumeOffset) { + this.consumeOffset = consumeOffset; + } + + public long getSeekOffset() { + return seekOffset; + } + + public void setSeekOffset(long seekOffset) { + this.seekOffset = seekOffset; + } + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java index edc2647a5f1d9c32503910dd5cf2aeeb7588deff..e440bd98a9c97dfe91063c37ef6fe626b4a4e656 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java @@ -374,6 +374,7 @@ public class ConsumeMessageOrderlyService implements ConsumeMessageService { MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes())); MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes())); + MessageAccessor.clearProperty(newMsg, MessageConst.PROPERTY_TRANSACTION_PREPARED); newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); this.defaultMQPushConsumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getDefaultMQProducer().send(newMsg); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..8ad7a6b44b7fe697ad937339e622595a061cd7fe --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -0,0 +1,1068 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.client.Validators; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.client.consumer.MessageQueueListener; +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.TopicMessageQueueChangeListener; +import org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.consumer.store.ReadOffsetType; +import org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.FilterMessageHook; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.ServiceState; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.common.filter.FilterAPI; +import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.NamespaceUtil; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.sysflag.PullSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingException; + +public class DefaultLitePullConsumerImpl implements MQConsumerInner { + + private final InternalLogger log = ClientLogger.getLog(); + + private final long consumerStartTimestamp = System.currentTimeMillis(); + + private final RPCHook rpcHook; + + private final ArrayList filterMessageHookList = new ArrayList(); + + private volatile ServiceState serviceState = ServiceState.CREATE_JUST; + + protected MQClientInstance mQClientFactory; + + private PullAPIWrapper pullAPIWrapper; + + private OffsetStore offsetStore; + + private RebalanceImpl rebalanceImpl = new RebalanceLitePullImpl(this); + + private enum SubscriptionType { + NONE, SUBSCRIBE, ASSIGN + } + + private static final String NOT_RUNNING_EXCEPTION_MESSAGE = "The consumer not running, please start it first."; + + private static final String SUBSCRIPTION_CONFILCT_EXCEPTION_MESSAGE = "Subscribe and assign are mutually exclusive."; + /** + * the type of subscription + */ + private SubscriptionType subscriptionType = SubscriptionType.NONE; + /** + * Delay some time when exception occur + */ + private long pullTimeDelayMillsWhenException = 1000; + /** + * Flow control interval + */ + private static final long PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL = 50; + /** + * Delay some time when suspend pull service + */ + private static final long PULL_TIME_DELAY_MILLS_WHEN_PAUSE = 1000; + + private DefaultLitePullConsumer defaultLitePullConsumer; + + private final ConcurrentMap taskTable = + new ConcurrentHashMap(); + + private AssignedMessageQueue assignedMessageQueue = new AssignedMessageQueue(); + + private final BlockingQueue consumeRequestCache = new LinkedBlockingQueue(); + + private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; + + private final ScheduledExecutorService scheduledExecutorService; + + private Map topicMessageQueueChangeListenerMap = new HashMap(); + + private Map> messageQueuesForTopic = new HashMap>(); + + private long consumeRequestFlowControlTimes = 0L; + + private long queueFlowControlTimes = 0L; + + private long queueMaxSpanFlowControlTimes = 0L; + + private long nextAutoCommitDeadline = -1L; + + private final MessageQueueLock messageQueueLock = new MessageQueueLock(); + + public DefaultLitePullConsumerImpl(final DefaultLitePullConsumer defaultLitePullConsumer, final RPCHook rpcHook) { + this.defaultLitePullConsumer = defaultLitePullConsumer; + this.rpcHook = rpcHook; + this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( + this.defaultLitePullConsumer.getPullThreadNums(), + new ThreadFactoryImpl("PullMsgThread-" + this.defaultLitePullConsumer.getConsumerGroup()) + ); + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "MonitorMessageQueueChangeThread"); + } + }); + this.pullTimeDelayMillsWhenException = defaultLitePullConsumer.getPullTimeDelayMillsWhenException(); + } + + private void checkServiceState() { + if (this.serviceState != ServiceState.RUNNING) + throw new IllegalStateException(NOT_RUNNING_EXCEPTION_MESSAGE); + } + + public void updateNameServerAddr(String newAddresses) { + this.mQClientFactory.getMQClientAPIImpl().updateNameServerAddressList(newAddresses); + } + + private synchronized void setSubscriptionType(SubscriptionType type) { + if (this.subscriptionType == SubscriptionType.NONE) + this.subscriptionType = type; + else if (this.subscriptionType != type) + throw new IllegalStateException(SUBSCRIPTION_CONFILCT_EXCEPTION_MESSAGE); + } + + private void updateAssignedMessageQueue(String topic, Set assignedMessageQueue) { + this.assignedMessageQueue.updateAssignedMessageQueue(topic, assignedMessageQueue); + } + + private void updatePullTask(String topic, Set mqNewSet) { + Iterator> it = this.taskTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + if (!mqNewSet.contains(next.getKey())) { + next.getValue().setCancelled(true); + it.remove(); + } + } + } + startPullTask(mqNewSet); + } + + class MessageQueueListenerImpl implements MessageQueueListener { + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + MessageModel messageModel = defaultLitePullConsumer.getMessageModel(); + switch (messageModel) { + case BROADCASTING: + updateAssignedMessageQueue(topic, mqAll); + updatePullTask(topic, mqAll); + break; + case CLUSTERING: + updateAssignedMessageQueue(topic, mqDivided); + updatePullTask(topic, mqDivided); + break; + default: + break; + } + } + } + + public synchronized void shutdown() { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + persistConsumerOffset(); + this.mQClientFactory.unregisterConsumer(this.defaultLitePullConsumer.getConsumerGroup()); + scheduledThreadPoolExecutor.shutdown(); + scheduledExecutorService.shutdown(); + this.mQClientFactory.shutdown(); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + log.info("the consumer [{}] shutdown OK", this.defaultLitePullConsumer.getConsumerGroup()); + break; + default: + break; + } + } + + public synchronized void start() throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + + this.checkConfig(); + + if (this.defaultLitePullConsumer.getMessageModel() == MessageModel.CLUSTERING) { + this.defaultLitePullConsumer.changeInstanceNameToPID(); + } + + initMQClientFactory(); + + initRebalanceImpl(); + + initPullAPIWrapper(); + + initOffsetStore(); + + mQClientFactory.start(); + + startScheduleTask(); + + this.serviceState = ServiceState.RUNNING; + + log.info("the consumer [{}] start OK", this.defaultLitePullConsumer.getConsumerGroup()); + + operateAfterRunning(); + + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The PullConsumer service state not OK, maybe started once, " + + this.serviceState + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + default: + break; + } + } + + private void initMQClientFactory() throws MQClientException { + this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultLitePullConsumer, this.rpcHook); + boolean registerOK = mQClientFactory.registerConsumer(this.defaultLitePullConsumer.getConsumerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + + throw new MQClientException("The consumer group[" + this.defaultLitePullConsumer.getConsumerGroup() + + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), + null); + } + } + + private void initRebalanceImpl() { + this.rebalanceImpl.setConsumerGroup(this.defaultLitePullConsumer.getConsumerGroup()); + this.rebalanceImpl.setMessageModel(this.defaultLitePullConsumer.getMessageModel()); + this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultLitePullConsumer.getAllocateMessageQueueStrategy()); + this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); + } + + private void initPullAPIWrapper() { + this.pullAPIWrapper = new PullAPIWrapper( + mQClientFactory, + this.defaultLitePullConsumer.getConsumerGroup(), isUnitMode()); + this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); + } + + private void initOffsetStore() throws MQClientException { + if (this.defaultLitePullConsumer.getOffsetStore() != null) { + this.offsetStore = this.defaultLitePullConsumer.getOffsetStore(); + } else { + switch (this.defaultLitePullConsumer.getMessageModel()) { + case BROADCASTING: + this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultLitePullConsumer.getConsumerGroup()); + break; + case CLUSTERING: + this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultLitePullConsumer.getConsumerGroup()); + break; + default: + break; + } + this.defaultLitePullConsumer.setOffsetStore(this.offsetStore); + } + this.offsetStore.load(); + } + + private void startScheduleTask() { + scheduledExecutorService.scheduleAtFixedRate( + new Runnable() { + @Override + public void run() { + try { + fetchTopicMessageQueuesAndCompare(); + } catch (Exception e) { + log.error("ScheduledTask fetchMessageQueuesAndCompare exception", e); + } + } + }, 1000 * 10, this.getDefaultLitePullConsumer().getTopicMetadataCheckIntervalMillis(), TimeUnit.MILLISECONDS); + } + + private void operateAfterRunning() throws MQClientException { + // If subscribe function invoke before start function, then update topic subscribe info after initialization. + if (subscriptionType == SubscriptionType.SUBSCRIBE) { + updateTopicSubscribeInfoWhenSubscriptionChanged(); + } + // If assign function invoke before start function, then update pull task after initialization. + if (subscriptionType == SubscriptionType.ASSIGN) { + updateAssignPullTask(assignedMessageQueue.messageQueues()); + } + + for (String topic : topicMessageQueueChangeListenerMap.keySet()) { + Set messageQueues = fetchMessageQueues(topic); + messageQueuesForTopic.put(topic, messageQueues); + } + this.mQClientFactory.checkClientInBroker(); + } + + private void checkConfig() throws MQClientException { + // Check consumerGroup + Validators.checkGroup(this.defaultLitePullConsumer.getConsumerGroup()); + + // Check consumerGroup name is not equal default consumer group name. + if (this.defaultLitePullConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) { + throw new MQClientException( + "consumerGroup can not equal " + + MixAll.DEFAULT_CONSUMER_GROUP + + ", please specify another one." + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // Check messageModel is not null. + if (null == this.defaultLitePullConsumer.getMessageModel()) { + throw new MQClientException( + "messageModel is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // Check allocateMessageQueueStrategy is not null + if (null == this.defaultLitePullConsumer.getAllocateMessageQueueStrategy()) { + throw new MQClientException( + "allocateMessageQueueStrategy is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + if (this.defaultLitePullConsumer.getConsumerTimeoutMillisWhenSuspend() < this.defaultLitePullConsumer.getBrokerSuspendMaxTimeMillis()) { + throw new MQClientException( + "Long polling mode, the consumer consumerTimeoutMillisWhenSuspend must greater than brokerSuspendMaxTimeMillis" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + } + + public PullAPIWrapper getPullAPIWrapper() { + return pullAPIWrapper; + } + + private void startPullTask(Collection mqSet) { + for (MessageQueue messageQueue : mqSet) { + if (!this.taskTable.containsKey(messageQueue)) { + PullTaskImpl pullTask = new PullTaskImpl(messageQueue); + this.taskTable.put(messageQueue, pullTask); + this.scheduledThreadPoolExecutor.schedule(pullTask, 0, TimeUnit.MILLISECONDS); + } + } + } + + private void updateAssignPullTask(Collection mqNewSet) { + Iterator> it = this.taskTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (!mqNewSet.contains(next.getKey())) { + next.getValue().setCancelled(true); + it.remove(); + } + } + + startPullTask(mqNewSet); + } + + private void updateTopicSubscribeInfoWhenSubscriptionChanged() { + Map subTable = rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + for (final Map.Entry entry : subTable.entrySet()) { + final String topic = entry.getKey(); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + } + } + } + + public synchronized void subscribe(String topic, String subExpression) throws MQClientException { + try { + if (topic == null || topic.equals("")) { + throw new IllegalArgumentException("Topic can not be null or empty."); + } + setSubscriptionType(SubscriptionType.SUBSCRIBE); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(defaultLitePullConsumer.getConsumerGroup(), + topic, subExpression); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + this.defaultLitePullConsumer.setMessageQueueListener(new MessageQueueListenerImpl()); + assignedMessageQueue.setRebalanceImpl(this.rebalanceImpl); + if (serviceState == ServiceState.RUNNING) { + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + updateTopicSubscribeInfoWhenSubscriptionChanged(); + } + } catch (Exception e) { + throw new MQClientException("subscribe exception", e); + } + } + + public synchronized void subscribe(String topic, MessageSelector messageSelector) throws MQClientException { + try { + if (topic == null || topic.equals("")) { + throw new IllegalArgumentException("Topic can not be null or empty."); + } + setSubscriptionType(SubscriptionType.SUBSCRIBE); + if (messageSelector == null) { + subscribe(topic, SubscriptionData.SUB_ALL); + return; + } + SubscriptionData subscriptionData = FilterAPI.build(topic, + messageSelector.getExpression(), messageSelector.getExpressionType()); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + this.defaultLitePullConsumer.setMessageQueueListener(new MessageQueueListenerImpl()); + assignedMessageQueue.setRebalanceImpl(this.rebalanceImpl); + if (serviceState == ServiceState.RUNNING) { + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + updateTopicSubscribeInfoWhenSubscriptionChanged(); + } + } catch (Exception e) { + throw new MQClientException("subscribe exception", e); + } + } + + public synchronized void unsubscribe(final String topic) { + this.rebalanceImpl.getSubscriptionInner().remove(topic); + removePullTaskCallback(topic); + assignedMessageQueue.removeAssignedMessageQueue(topic); + } + + public synchronized void assign(Collection messageQueues) { + if (messageQueues == null || messageQueues.isEmpty()) { + throw new IllegalArgumentException("Message queues can not be null or empty."); + } + setSubscriptionType(SubscriptionType.ASSIGN); + assignedMessageQueue.updateAssignedMessageQueue(messageQueues); + if (serviceState == ServiceState.RUNNING) { + updateAssignPullTask(messageQueues); + } + } + + private void maybeAutoCommit() { + long now = System.currentTimeMillis(); + if (now >= nextAutoCommitDeadline) { + commitAll(); + nextAutoCommitDeadline = now + defaultLitePullConsumer.getAutoCommitIntervalMillis(); + } + } + + public synchronized List poll(long timeout) { + try { + checkServiceState(); + if (timeout < 0) + throw new IllegalArgumentException("Timeout must not be negative"); + + if (defaultLitePullConsumer.isAutoCommit()) { + maybeAutoCommit(); + } + long endTime = System.currentTimeMillis() + timeout; + + ConsumeRequest consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS); + + if (endTime - System.currentTimeMillis() > 0) { + while (consumeRequest != null && consumeRequest.getProcessQueue().isDropped()) { + consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS); + if (endTime - System.currentTimeMillis() <= 0) + break; + } + } + + if (consumeRequest != null && !consumeRequest.getProcessQueue().isDropped()) { + List messages = consumeRequest.getMessageExts(); + long offset = consumeRequest.getProcessQueue().removeMessage(messages); + assignedMessageQueue.updateConsumeOffset(consumeRequest.getMessageQueue(), offset); + //If namespace not null , reset Topic without namespace. + this.resetTopic(messages); + return messages; + } + } catch (InterruptedException ignore) { + + } + + return Collections.emptyList(); + } + + public void pause(Collection messageQueues) { + assignedMessageQueue.pause(messageQueues); + } + + public void resume(Collection messageQueues) { + assignedMessageQueue.resume(messageQueues); + } + + public synchronized void seek(MessageQueue messageQueue, long offset) throws MQClientException { + if (!assignedMessageQueue.messageQueues().contains(messageQueue)) { + if (subscriptionType == SubscriptionType.SUBSCRIBE) { + throw new MQClientException("The message queue is not in assigned list, may be rebalancing, message queue: " + messageQueue, null); + } else { + throw new MQClientException("The message queue is not in assigned list, message queue: " + messageQueue, null); + } + } + long minOffset = minOffset(messageQueue); + long maxOffset = maxOffset(messageQueue); + if (offset < minOffset || offset > maxOffset) { + throw new MQClientException("Seek offset illegal, seek offset = " + offset + ", min offset = " + minOffset + ", max offset = " + maxOffset, null); + } + final Object objLock = messageQueueLock.fetchLockObject(messageQueue); + synchronized (objLock) { + assignedMessageQueue.setSeekOffset(messageQueue, offset); + clearMessageQueueInCache(messageQueue); + } + } + + public void seekToBegin(MessageQueue messageQueue) throws MQClientException { + long begin = minOffset(messageQueue); + this.seek(messageQueue, begin); + } + + public void seekToEnd(MessageQueue messageQueue) throws MQClientException { + long end = maxOffset(messageQueue); + this.seek(messageQueue, end); + } + + private long maxOffset(MessageQueue messageQueue) throws MQClientException { + checkServiceState(); + return this.mQClientFactory.getMQAdminImpl().maxOffset(messageQueue); + } + + private long minOffset(MessageQueue messageQueue) throws MQClientException { + checkServiceState(); + return this.mQClientFactory.getMQAdminImpl().minOffset(messageQueue); + } + + private void removePullTaskCallback(final String topic) { + removePullTask(topic); + } + + private void removePullTask(final String topic) { + Iterator> it = this.taskTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + next.getValue().setCancelled(true); + it.remove(); + } + } + } + + public synchronized void commitAll() { + try { + for (MessageQueue messageQueue : assignedMessageQueue.messageQueues()) { + long consumerOffset = assignedMessageQueue.getConsumerOffset(messageQueue); + if (consumerOffset != -1) { + ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); + if (processQueue != null && !processQueue.isDropped()) { + updateConsumeOffset(messageQueue, consumerOffset); + } + } + } + if (defaultLitePullConsumer.getMessageModel() == MessageModel.BROADCASTING) { + offsetStore.persistAll(assignedMessageQueue.messageQueues()); + } + } catch (Exception e) { + log.error("An error occurred when update consume offset Automatically."); + } + } + + private void updatePullOffset(MessageQueue messageQueue, long nextPullOffset) { + if (assignedMessageQueue.getSeekOffset(messageQueue) == -1) { + assignedMessageQueue.updatePullOffset(messageQueue, nextPullOffset); + } + } + + private void submitConsumeRequest(ConsumeRequest consumeRequest) { + try { + consumeRequestCache.put(consumeRequest); + } catch (InterruptedException e) { + log.error("Submit consumeRequest error", e); + } + } + + private long fetchConsumeOffset(MessageQueue messageQueue) { + checkServiceState(); + long offset = this.rebalanceImpl.computePullFromWhere(messageQueue); + return offset; + } + + public long committed(MessageQueue messageQueue) throws MQClientException { + checkServiceState(); + long offset = this.offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE); + if (offset == -2) + throw new MQClientException("Fetch consume offset from broker exception", null); + return offset; + } + + private void clearMessageQueueInCache(MessageQueue messageQueue) { + ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); + if (processQueue != null) { + processQueue.clear(); + } + Iterator iter = consumeRequestCache.iterator(); + while (iter.hasNext()) { + if (iter.next().getMessageQueue().equals(messageQueue)) + iter.remove(); + } + } + + private long nextPullOffset(MessageQueue messageQueue) { + long offset = -1; + long seekOffset = assignedMessageQueue.getSeekOffset(messageQueue); + if (seekOffset != -1) { + offset = seekOffset; + assignedMessageQueue.updateConsumeOffset(messageQueue, offset); + assignedMessageQueue.setSeekOffset(messageQueue, -1); + } else { + offset = assignedMessageQueue.getPullOffset(messageQueue); + if (offset == -1) { + offset = fetchConsumeOffset(messageQueue); + } + } + return offset; + } + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + checkServiceState(); + return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + + public class PullTaskImpl implements Runnable { + private final MessageQueue messageQueue; + private volatile boolean cancelled = false; + + public PullTaskImpl(final MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + @Override + public void run() { + + if (!this.isCancelled()) { + + if (assignedMessageQueue.isPaused(messageQueue)) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_PAUSE, TimeUnit.MILLISECONDS); + log.debug("Message Queue: {} has been paused!", messageQueue); + return; + } + + ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); + + if (processQueue == null && processQueue.isDropped()) { + log.info("The message queue not be able to poll, because it's dropped. group={}, messageQueue={}", defaultLitePullConsumer.getConsumerGroup(), this.messageQueue); + return; + } + + if (consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchSize() > defaultLitePullConsumer.getPullThresholdForAll()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((consumeRequestFlowControlTimes++ % 1000) == 0) + log.warn("The consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes); + return; + } + + long cachedMessageCount = processQueue.getMsgCount().get(); + long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024); + + if (cachedMessageCount > defaultLitePullConsumer.getPullThresholdForQueue()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((queueFlowControlTimes++ % 1000) == 0) { + log.warn( + "The cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", + defaultLitePullConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes); + } + return; + } + + if (cachedMessageSizeInMiB > defaultLitePullConsumer.getPullThresholdSizeForQueue()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((queueFlowControlTimes++ % 1000) == 0) { + log.warn( + "The cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", + defaultLitePullConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes); + } + return; + } + + if (processQueue.getMaxSpan() > defaultLitePullConsumer.getConsumeMaxSpan()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) { + log.warn( + "The queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, flowControlTimes={}", + processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), queueMaxSpanFlowControlTimes); + } + return; + } + + long offset = nextPullOffset(messageQueue); + long pullDelayTimeMills = 0; + try { + SubscriptionData subscriptionData; + if (subscriptionType == SubscriptionType.SUBSCRIBE) { + String topic = this.messageQueue.getTopic(); + subscriptionData = rebalanceImpl.getSubscriptionInner().get(topic); + } else { + String topic = this.messageQueue.getTopic(); + subscriptionData = FilterAPI.buildSubscriptionData(defaultLitePullConsumer.getConsumerGroup(), + topic, SubscriptionData.SUB_ALL); + } + + PullResult pullResult = pull(messageQueue, subscriptionData, offset, defaultLitePullConsumer.getPullBatchSize()); + + switch (pullResult.getPullStatus()) { + case FOUND: + final Object objLock = messageQueueLock.fetchLockObject(messageQueue); + synchronized (objLock) { + if (pullResult.getMsgFoundList() != null && !pullResult.getMsgFoundList().isEmpty() && assignedMessageQueue.getSeekOffset(messageQueue) == -1) { + processQueue.putMessage(pullResult.getMsgFoundList()); + submitConsumeRequest(new ConsumeRequest(pullResult.getMsgFoundList(), messageQueue, processQueue)); + } + } + break; + case OFFSET_ILLEGAL: + log.warn("The pull request offset illegal, {}", pullResult.toString()); + break; + default: + break; + } + updatePullOffset(messageQueue, pullResult.getNextBeginOffset()); + } catch (Throwable e) { + pullDelayTimeMills = pullTimeDelayMillsWhenException; + log.error("An error occurred in pull message process.", e); + } + + if (!this.isCancelled()) { + scheduledThreadPoolExecutor.schedule(this, pullDelayTimeMills, TimeUnit.MILLISECONDS); + } else { + log.warn("The Pull Task is cancelled after doPullTask, {}", messageQueue); + } + } + } + + public boolean isCancelled() { + return cancelled; + } + + public void setCancelled(boolean cancelled) { + this.cancelled = cancelled; + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + } + + private PullResult pull(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return pull(mq, subscriptionData, offset, maxNums, this.defaultLitePullConsumer.getConsumerPullTimeoutMillis()); + } + + private PullResult pull(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, timeout); + } + + private PullResult pullSyncImpl(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums, + boolean block, + long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + + if (null == mq) { + throw new MQClientException("mq is null", null); + } + + if (offset < 0) { + throw new MQClientException("offset < 0", null); + } + + if (maxNums <= 0) { + throw new MQClientException("maxNums <= 0", null); + } + + int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false, true); + + long timeoutMillis = block ? this.defaultLitePullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout; + + boolean isTagType = ExpressionType.isTagType(subscriptionData.getExpressionType()); + PullResult pullResult = this.pullAPIWrapper.pullKernelImpl( + mq, + subscriptionData.getSubString(), + subscriptionData.getExpressionType(), + isTagType ? 0L : subscriptionData.getSubVersion(), + offset, + maxNums, + sysFlag, + 0, + this.defaultLitePullConsumer.getBrokerSuspendMaxTimeMillis(), + timeoutMillis, + CommunicationMode.SYNC, + null + ); + this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); + return pullResult; + } + + private void resetTopic(List msgList) { + if (null == msgList || msgList.size() == 0) { + return; + } + + //If namespace not null , reset Topic without namespace. + for (MessageExt messageExt : msgList) { + if (null != this.defaultLitePullConsumer.getNamespace()) { + messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), this.defaultLitePullConsumer.getNamespace())); + } + } + + } + + public void updateConsumeOffset(MessageQueue mq, long offset) { + checkServiceState(); + this.offsetStore.updateOffset(mq, offset, false); + } + + @Override + public String groupName() { + return this.defaultLitePullConsumer.getConsumerGroup(); + } + + @Override + public MessageModel messageModel() { + return this.defaultLitePullConsumer.getMessageModel(); + } + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_ACTIVELY; + } + + @Override + public ConsumeFromWhere consumeFromWhere() { + return ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET; + } + + @Override + public Set subscriptions() { + Set subSet = new HashSet(); + + subSet.addAll(this.rebalanceImpl.getSubscriptionInner().values()); + + return subSet; + } + + @Override + public void doRebalance() { + if (this.rebalanceImpl != null) { + this.rebalanceImpl.doRebalance(false); + } + } + + @Override + public void persistConsumerOffset() { + try { + checkServiceState(); + Set mqs = new HashSet(); + if (this.subscriptionType == SubscriptionType.SUBSCRIBE) { + Set allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet(); + mqs.addAll(allocateMq); + } else if (this.subscriptionType == SubscriptionType.ASSIGN) { + Set assignedMessageQueue = this.assignedMessageQueue.getAssignedMessageQueues(); + mqs.addAll(assignedMessageQueue); + } + this.offsetStore.persistAll(mqs); + } catch (Exception e) { + log.error("Persist consumer offset error for group: {} ", this.defaultLitePullConsumer.getConsumerGroup(), e); + } + } + + @Override + public void updateTopicSubscribeInfo(String topic, Set info) { + Map subTable = this.rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + this.rebalanceImpl.getTopicSubscribeInfoTable().put(topic, info); + } + } + } + + @Override + public boolean isSubscribeTopicNeedUpdate(String topic) { + Map subTable = this.rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic); + } + } + + return false; + } + + @Override + public boolean isUnitMode() { + return this.defaultLitePullConsumer.isUnitMode(); + } + + @Override + public ConsumerRunningInfo consumerRunningInfo() { + ConsumerRunningInfo info = new ConsumerRunningInfo(); + + Properties prop = MixAll.object2Properties(this.defaultLitePullConsumer); + prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, String.valueOf(this.consumerStartTimestamp)); + info.setProperties(prop); + + info.getSubscriptionSet().addAll(this.subscriptions()); + return info; + } + + private void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + this.offsetStore.updateConsumeOffsetToBroker(mq, offset, isOneway); + } + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + public DefaultLitePullConsumer getDefaultLitePullConsumer() { + return defaultLitePullConsumer; + } + + public Set fetchMessageQueues(String topic) throws MQClientException { + checkServiceState(); + Set result = this.mQClientFactory.getMQAdminImpl().fetchSubscribeMessageQueues(topic); + return parseMessageQueues(result); + } + + private synchronized void fetchTopicMessageQueuesAndCompare() throws MQClientException { + for (Map.Entry entry : topicMessageQueueChangeListenerMap.entrySet()) { + String topic = entry.getKey(); + TopicMessageQueueChangeListener topicMessageQueueChangeListener = entry.getValue(); + Set oldMessageQueues = messageQueuesForTopic.get(topic); + Set newMessageQueues = fetchMessageQueues(topic); + boolean isChanged = !isSetEqual(newMessageQueues, oldMessageQueues); + if (isChanged) { + messageQueuesForTopic.put(topic, newMessageQueues); + if (topicMessageQueueChangeListener != null) { + topicMessageQueueChangeListener.onChanged(topic, newMessageQueues); + } + } + } + } + + private boolean isSetEqual(Set set1, Set set2) { + if (set1 == null && set2 == null) { + return true; + } + + if (set1 == null || set2 == null || set1.size() != set2.size() + || set1.size() == 0 || set2.size() == 0) { + return false; + } + + Iterator iter = set2.iterator(); + boolean isEqual = true; + while (iter.hasNext()) { + if (!set1.contains(iter.next())) { + isEqual = false; + } + } + return isEqual; + } + + public synchronized void registerTopicMessageQueueChangeListener(String topic, + TopicMessageQueueChangeListener listener) throws MQClientException { + if (topic == null || listener == null) { + throw new MQClientException("Topic or listener is null", null); + } + if (topicMessageQueueChangeListenerMap.containsKey(topic)) { + log.warn("Topic {} had been registered, new listener will overwrite the old one", topic); + } + topicMessageQueueChangeListenerMap.put(topic, listener); + if (this.serviceState == ServiceState.RUNNING) { + Set messageQueues = fetchMessageQueues(topic); + messageQueuesForTopic.put(topic, messageQueues); + } + } + + private Set parseMessageQueues(Set queueSet) { + Set resultQueues = new HashSet(); + for (MessageQueue messageQueue : queueSet) { + String userTopic = NamespaceUtil.withoutNamespace(messageQueue.getTopic(), + this.defaultLitePullConsumer.getNamespace()); + resultQueues.add(new MessageQueue(userTopic, messageQueue.getBrokerName(), messageQueue.getQueueId())); + } + return resultQueues; + } + + public class ConsumeRequest { + private final List messageExts; + private final MessageQueue messageQueue; + private final ProcessQueue processQueue; + + public ConsumeRequest(final List messageExts, final MessageQueue messageQueue, + final ProcessQueue processQueue) { + this.messageExts = messageExts; + this.messageQueue = messageQueue; + this.processQueue = processQueue; + } + + public List getMessageExts() { + return messageExts; + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + } + + public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) { + this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index 598dc94cc26a5984345e4cf62b6dbd53028263c4..afd72a08002c6c9124feb6260f011e9810407842 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -66,6 +66,11 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; +/** + * This class will be removed in 2022, and a better implementation {@link DefaultLitePullConsumerImpl} is recommend to use + * in the scenario of actively pulling messages. + */ +@Deprecated public class DefaultMQPullConsumerImpl implements MQConsumerInner { private final InternalLogger log = ClientLogger.getLog(); private final DefaultMQPullConsumer defaultMQPullConsumer; @@ -74,7 +79,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { private final ArrayList consumeMessageHookList = new ArrayList(); private final ArrayList filterMessageHookList = new ArrayList(); private volatile ServiceState serviceState = ServiceState.CREATE_JUST; - private MQClientInstance mQClientFactory; + protected MQClientInstance mQClientFactory; private PullAPIWrapper pullAPIWrapper; private OffsetStore offsetStore; private RebalanceImpl rebalanceImpl = new RebalancePullImpl(this); @@ -629,7 +634,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { this.defaultMQPullConsumer.changeInstanceNameToPID(); } - this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPullConsumer, this.rpcHook); + this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQPullConsumer, this.rpcHook); this.rebalanceImpl.setConsumerGroup(this.defaultMQPullConsumer.getConsumerGroup()); this.rebalanceImpl.setMessageModel(this.defaultMQPullConsumer.getMessageModel()); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 48fe41ada6d6c376cec49d589b2cf6881e72f2c7..25a81a0e755e60e968ae281d4390cff5b77edcc7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -83,7 +83,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { /** * Delay some time when exception occur */ - private static final long PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION = 3000; + private long pullTimeDelayMillsWhenException = 3000; /** * Flow control interval */ @@ -115,6 +115,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { public DefaultMQPushConsumerImpl(DefaultMQPushConsumer defaultMQPushConsumer, RPCHook rpcHook) { this.defaultMQPushConsumer = defaultMQPushConsumer; this.rpcHook = rpcHook; + this.pullTimeDelayMillsWhenException = defaultMQPushConsumer.getPullTimeDelayMillsWhenException(); } public void registerFilterMessageHook(final FilterMessageHook hook) { @@ -222,7 +223,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { this.makeSureStateOK(); } catch (MQClientException e) { log.warn("pullMessage exception, consumer state not ok", e); - this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); return; } @@ -282,7 +283,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { pullRequest.setNextOffset(offset); } } else { - this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); log.info("pull message later because not locked in broker, {}", pullRequest); return; } @@ -290,7 +291,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); if (null == subscriptionData) { - this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); log.warn("find the consumer's subscription failed, {}", pullRequest); return; } @@ -397,7 +398,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { log.warn("execute the pull request exception", e); } - DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); } }; @@ -444,7 +445,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { ); } catch (Exception e) { log.error("pullKernelImpl exception", e); - this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); } } @@ -527,6 +528,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1)); MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes())); + MessageAccessor.clearProperty(newMsg, MessageConst.PROPERTY_TRANSACTION_PREPARED); newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); this.mQClientFactory.getDefaultMQProducer().send(newMsg); @@ -579,7 +581,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { this.defaultMQPushConsumer.changeInstanceNameToPID(); } - this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook); + this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook); this.rebalanceImpl.setConsumerGroup(this.defaultMQPushConsumer.getConsumerGroup()); this.rebalanceImpl.setMessageModel(this.defaultMQPushConsumer.getMessageModel()); @@ -1168,4 +1170,8 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { this.consumeMessageService = consumeMessageService; } + + public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) { + this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java index 0a52817ced7d9350c9a38dfc3fe358c193307364..092da9aa33e1997639ed311e822397bec7c1d053 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java @@ -26,6 +26,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; + import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.logging.InternalLogger; @@ -431,4 +432,5 @@ public class ProcessQueue { public void setLastConsumeTimestamp(long lastConsumeTimestamp) { this.lastConsumeTimestamp = lastConsumeTimestamp; } + } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java index 9ad07c7e4433548d64693c294e2b1bf58ea0e530..b8972a92e8fdb3402b92c7c1b1a8015cb38a127b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -40,9 +40,6 @@ import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -/** - * Base class for rebalance algorithm - */ public abstract class RebalanceImpl { protected static final InternalLogger log = ClientLogger.getLog(); protected final ConcurrentMap processQueueTable = new ConcurrentHashMap(64); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..9d1ea7492eea2ffc662defea11f4fe5d1bf84f38 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import java.util.List; +import java.util.Set; +import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import org.apache.rocketmq.client.consumer.MessageQueueListener; +import org.apache.rocketmq.client.consumer.store.ReadOffsetType; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; + +public class RebalanceLitePullImpl extends RebalanceImpl { + + private final DefaultLitePullConsumerImpl litePullConsumerImpl; + + public RebalanceLitePullImpl(DefaultLitePullConsumerImpl litePullConsumerImpl) { + this(null, null, null, null, litePullConsumerImpl); + } + + public RebalanceLitePullImpl(String consumerGroup, MessageModel messageModel, + AllocateMessageQueueStrategy allocateMessageQueueStrategy, + MQClientInstance mQClientFactory, DefaultLitePullConsumerImpl litePullConsumerImpl) { + super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory); + this.litePullConsumerImpl = litePullConsumerImpl; + } + + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + MessageQueueListener messageQueueListener = this.litePullConsumerImpl.getDefaultLitePullConsumer().getMessageQueueListener(); + if (messageQueueListener != null) { + try { + messageQueueListener.messageQueueChanged(topic, mqAll, mqDivided); + } catch (Throwable e) { + log.error("messageQueueChanged exception", e); + } + } + } + + @Override + public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) { + this.litePullConsumerImpl.getOffsetStore().persist(mq); + this.litePullConsumerImpl.getOffsetStore().removeOffset(mq); + return true; + } + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_ACTIVELY; + } + + @Override + public void removeDirtyOffset(final MessageQueue mq) { + this.litePullConsumerImpl.getOffsetStore().removeOffset(mq); + } + + @Override + public long computePullFromWhere(MessageQueue mq) { + ConsumeFromWhere consumeFromWhere = litePullConsumerImpl.getDefaultLitePullConsumer().getConsumeFromWhere(); + long result = -1; + switch (consumeFromWhere) { + case CONSUME_FROM_LAST_OFFSET: { + long lastOffset = litePullConsumerImpl.getOffsetStore().readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE); + if (lastOffset >= 0) { + result = lastOffset; + } else if (-1 == lastOffset) { + if (mq.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { // First start, no offset + result = 0L; + } else { + try { + result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } catch (MQClientException e) { + result = -1; + } + } + } else { + result = -1; + } + break; + } + case CONSUME_FROM_FIRST_OFFSET: { + long lastOffset = litePullConsumerImpl.getOffsetStore().readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE); + if (lastOffset >= 0) { + result = lastOffset; + } else if (-1 == lastOffset) { + result = 0L; + } else { + result = -1; + } + break; + } + case CONSUME_FROM_TIMESTAMP: { + long lastOffset = litePullConsumerImpl.getOffsetStore().readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE); + if (lastOffset >= 0) { + result = lastOffset; + } else if (-1 == lastOffset) { + if (mq.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + try { + result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } catch (MQClientException e) { + result = -1; + } + } else { + try { + long timestamp = UtilAll.parseDate(this.litePullConsumerImpl.getDefaultLitePullConsumer().getConsumeTimestamp(), + UtilAll.YYYYMMDDHHMMSS).getTime(); + result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } catch (MQClientException e) { + result = -1; + } + } + } else { + result = -1; + } + break; + } + } + return result; + } + + @Override + public void dispatchPullRequest(List pullRequestList) { + } + +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 2a4fb7dfa6cedc44079425b5c455b85db56e9e1c..bbd2eecb1c3785f7520ade1d830b8f2ce19cb1e4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -246,10 +246,6 @@ public class MQClientInstance { log.info("the client factory [{}] start OK", this.clientId); this.serviceState = ServiceState.RUNNING; break; - case RUNNING: - break; - case SHUTDOWN_ALREADY: - break; case START_FAILED: throw new MQClientException("The Factory object[" + this.getClientId() + "] has been created before, and failed.", null); default: @@ -366,7 +362,6 @@ public class MQClientInstance { } /** - * * @param offsetTable * @param namespace * @return newOffsetTable @@ -385,6 +380,7 @@ public class MQClientInstance { return newOffsetTable; } + /** * Remove offline broker */ @@ -676,10 +672,13 @@ public class MQClientInstance { } else { log.warn("updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}", topic); } - } catch (Exception e) { + } catch (MQClientException e) { if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) && !topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { log.warn("updateTopicRouteInfoFromNameServer Exception", e); } + } catch (RemotingException e) { + log.error("updateTopicRouteInfoFromNameServer Exception", e); + throw new IllegalStateException(e); } finally { this.lockNamesrv.unlock(); } @@ -743,9 +742,10 @@ public class MQClientInstance { return false; } + /** - * This method will be removed in the version 5.0.0,because filterServer was removed,and method subscribe(final String topic, final MessageSelector messageSelector) - * is recommended. + * This method will be removed in the version 5.0.0,because filterServer was removed,and method + * subscribe(final String topic, final MessageSelector messageSelector) is recommended. */ @Deprecated private void uploadFilterClassToAllFilterServer(final String consumerGroup, final String fullClassName, diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 62aaef3b13ff9996c65a712867120dc4321360e5..fca50cc565ceb398d66f5c1d28d98467396db221 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -24,6 +24,8 @@ import java.util.HashSet; import java.util.List; import java.util.Random; import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -39,6 +41,7 @@ import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.common.ClientErrorCode; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.hook.CheckForbiddenContext; import org.apache.rocketmq.client.hook.CheckForbiddenHook; import org.apache.rocketmq.client.hook.SendMessageContext; @@ -52,6 +55,9 @@ import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.LocalTransactionExecuter; import org.apache.rocketmq.client.producer.LocalTransactionState; import org.apache.rocketmq.client.producer.MessageQueueSelector; +import org.apache.rocketmq.client.producer.RequestCallback; +import org.apache.rocketmq.client.producer.RequestFutureTable; +import org.apache.rocketmq.client.producer.RequestResponseFuture; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; @@ -79,6 +85,7 @@ import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHe import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.utils.CorrelationIdUtil; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -95,17 +102,16 @@ public class DefaultMQProducerImpl implements MQProducerInner { new ConcurrentHashMap(); private final ArrayList sendMessageHookList = new ArrayList(); private final RPCHook rpcHook; + private final BlockingQueue asyncSenderThreadPoolQueue; + private final ExecutorService defaultAsyncSenderExecutor; + private final Timer timer = new Timer("RequestHouseKeepingService", true); protected BlockingQueue checkRequestQueue; protected ExecutorService checkExecutor; private ServiceState serviceState = ServiceState.CREATE_JUST; private MQClientInstance mQClientFactory; private ArrayList checkForbiddenHookList = new ArrayList(); private int zipCompressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5")); - private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); - - private final BlockingQueue asyncSenderThreadPoolQueue; - private final ExecutorService defaultAsyncSenderExecutor; private ExecutorService asyncSenderExecutor; public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { @@ -180,7 +186,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { this.defaultMQProducer.changeInstanceNameToPID(); } - this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQProducer, rpcHook); + this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQProducer, rpcHook); boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this); if (!registerOK) { @@ -212,6 +218,17 @@ public class DefaultMQProducerImpl implements MQProducerInner { } this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + + this.timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + try { + RequestFutureTable.scanExpiredRequest(); + } catch (Throwable e) { + log.error("scan RequestFutureTable exception", e); + } + } + }, 1000 * 3, 1000); } private void checkConfig() throws MQClientException { @@ -271,6 +288,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { /** * This method will be removed in the version 5.0.0 and getCheckListener is recommended. + * * @return */ @Override @@ -464,13 +482,14 @@ public class DefaultMQProducerImpl implements MQProducerInner { * DEFAULT ASYNC ------------------------------------------------------- */ public void send(Message msg, - SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { + SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { send(msg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); } /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. - * A new one will be provided in next version + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * * @param msg * @param sendCallback * @param timeout the sendCallback will be invoked at most time @@ -505,7 +524,6 @@ public class DefaultMQProducerImpl implements MQProducerInner { } - public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); } @@ -514,6 +532,15 @@ public class DefaultMQProducerImpl implements MQProducerInner { this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation); } + private void validateNameServerSetting() throws MQClientException { + List nsList = this.getmQClientFactory().getMQClientAPIImpl().getNameServerAddressList(); + if (null == nsList || nsList.isEmpty()) { + throw new MQClientException( + "No name server address, please set it." + FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL), null).setResponseCode(ClientErrorCode.NO_NAME_SERVER_EXCEPTION); + } + + } + private SendResult sendDefaultImpl( Message msg, final CommunicationMode communicationMode, @@ -522,7 +549,6 @@ public class DefaultMQProducerImpl implements MQProducerInner { ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { this.makeSureStateOK(); Validators.checkMessage(msg, this.defaultMQProducer); - final long invokeID = random.nextLong(); long beginTimestampFirst = System.currentTimeMillis(); long beginTimestampPrev = beginTimestampFirst; @@ -653,13 +679,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { throw mqClientException; } - List nsList = this.getmQClientFactory().getMQClientAPIImpl().getNameServerAddressList(); - if (null == nsList || nsList.isEmpty()) { - throw new MQClientException( - "No name server address, please set it." + FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL), null).setResponseCode(ClientErrorCode.NO_NAME_SERVER_EXCEPTION); - } + validateNameServerSetting(); - throw new MQClientException("No route info of this topic, " + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO), + throw new MQClientException("No route info of this topic: " + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO), null).setResponseCode(ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION); } @@ -681,11 +703,11 @@ public class DefaultMQProducerImpl implements MQProducerInner { } private SendResult sendKernelImpl(final Message msg, - final MessageQueue mq, - final CommunicationMode communicationMode, - final SendCallback sendCallback, - final TopicPublishInfo topicPublishInfo, - final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + final MessageQueue mq, + final CommunicationMode communicationMode, + final SendCallback sendCallback, + final TopicPublishInfo topicPublishInfo, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { long beginStartTime = System.currentTimeMillis(); String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); if (null == brokerAddr) { @@ -990,8 +1012,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { } /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. - * A new one will be provided in next version + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * * @param msg * @param mq * @param sendCallback @@ -1105,6 +1128,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } } + validateNameServerSetting(); throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); } @@ -1117,8 +1141,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { } /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. - * A new one will be provided in next version + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * * @param msg * @param selector * @param arg @@ -1129,7 +1154,8 @@ public class DefaultMQProducerImpl implements MQProducerInner { * @throws InterruptedException */ @Deprecated - public void send(final Message msg, final MessageQueueSelector selector, final Object arg, final SendCallback sendCallback, final long timeout) + public void send(final Message msg, final MessageQueueSelector selector, final Object arg, + final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { final long beginStartTime = System.currentTimeMillis(); ExecutorService executor = this.getAsyncSenderExecutor(); @@ -1173,12 +1199,18 @@ public class DefaultMQProducerImpl implements MQProducerInner { } public TransactionSendResult sendMessageInTransaction(final Message msg, - final LocalTransactionExecuter localTransactionExecuter, final Object arg) + final LocalTransactionExecuter localTransactionExecuter, final Object arg) throws MQClientException { TransactionListener transactionListener = getCheckListener(); if (null == localTransactionExecuter && null == transactionListener) { throw new MQClientException("tranExecutor is null", null); } + + // ignore DelayTimeLevel parameter + if (msg.getDelayTimeLevel() != 0) { + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_DELAY_TIME_LEVEL); + } + Validators.checkMessage(msg, this.defaultMQProducer); SendResult sendResult = null; @@ -1310,6 +1342,233 @@ public class DefaultMQProducerImpl implements MQProducerInner { return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout); } + public Message request(Message msg, + long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginTimestamp = System.currentTimeMillis(); + prepareSendRequest(msg, timeout); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); + + try { + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, null); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); + + long cost = System.currentTimeMillis() - beginTimestamp; + this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setSendReqeustOk(false); + requestResponseFuture.putResponseMessage(null); + requestResponseFuture.setCause(e); + } + }, timeout - cost); + + Message responseMessage = requestResponseFuture.waitResponseMessage(timeout - cost); + if (responseMessage == null) { + if (requestResponseFuture.isSendRequestOk()) { + throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, + "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); + } else { + throw new MQClientException("send request message to <" + msg.getTopic() + "> fail", requestResponseFuture.getCause()); + } + } + return responseMessage; + } finally { + RequestFutureTable.getRequestFutureTable().remove(correlationId); + } + } + + public void request(Message msg, final RequestCallback requestCallback, long timeout) + throws RemotingException, InterruptedException, MQClientException, MQBrokerException { + long beginTimestamp = System.currentTimeMillis(); + prepareSendRequest(msg, timeout); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); + + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); + + long cost = System.currentTimeMillis() - beginTimestamp; + this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setCause(e); + requestFail(correlationId); + } + }, timeout - cost); + } + + public Message request(final Message msg, final MessageQueueSelector selector, final Object arg, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException, RequestTimeoutException { + long beginTimestamp = System.currentTimeMillis(); + prepareSendRequest(msg, timeout); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); + + try { + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, null); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); + + long cost = System.currentTimeMillis() - beginTimestamp; + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setSendReqeustOk(false); + requestResponseFuture.putResponseMessage(null); + requestResponseFuture.setCause(e); + } + }, timeout - cost); + + Message responseMessage = requestResponseFuture.waitResponseMessage(timeout - cost); + if (responseMessage == null) { + if (requestResponseFuture.isSendRequestOk()) { + throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, + "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); + } else { + throw new MQClientException("send request message to <" + msg.getTopic() + "> fail", requestResponseFuture.getCause()); + } + } + return responseMessage; + } finally { + RequestFutureTable.getRequestFutureTable().remove(correlationId); + } + } + + public void request(final Message msg, final MessageQueueSelector selector, final Object arg, + final RequestCallback requestCallback, final long timeout) + throws RemotingException, InterruptedException, MQClientException, MQBrokerException { + long beginTimestamp = System.currentTimeMillis(); + prepareSendRequest(msg, timeout); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); + + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); + + long cost = System.currentTimeMillis() - beginTimestamp; + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setCause(e); + requestFail(correlationId); + } + }, timeout - cost); + + } + + public Message request(final Message msg, final MessageQueue mq, final long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { + long beginTimestamp = System.currentTimeMillis(); + prepareSendRequest(msg, timeout); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); + + try { + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, null); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); + + long cost = System.currentTimeMillis() - beginTimestamp; + this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setSendReqeustOk(false); + requestResponseFuture.putResponseMessage(null); + requestResponseFuture.setCause(e); + } + }, null, timeout - cost); + + Message responseMessage = requestResponseFuture.waitResponseMessage(timeout - cost); + if (responseMessage == null) { + if (requestResponseFuture.isSendRequestOk()) { + throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, + "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); + } else { + throw new MQClientException("send request message to <" + msg.getTopic() + "> fail", requestResponseFuture.getCause()); + } + } + return responseMessage; + } finally { + RequestFutureTable.getRequestFutureTable().remove(correlationId); + } + } + + public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) + throws RemotingException, InterruptedException, MQClientException, MQBrokerException { + long beginTimestamp = System.currentTimeMillis(); + prepareSendRequest(msg, timeout); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); + + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); + + long cost = System.currentTimeMillis() - beginTimestamp; + this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setCause(e); + requestFail(correlationId); + } + }, null, timeout - cost); + } + + private void requestFail(final String correlationId) { + RequestResponseFuture responseFuture = RequestFutureTable.getRequestFutureTable().remove(correlationId); + if (responseFuture != null) { + responseFuture.setSendReqeustOk(false); + responseFuture.putResponseMessage(null); + try { + responseFuture.executeRequestCallback(); + } catch (Exception e) { + log.warn("execute requestCallback in requestFail, and callback throw", e); + } + } + } + + private void prepareSendRequest(final Message msg, long timeout) { + String correlationId = CorrelationIdUtil.createCorrelationId(); + String requestClientId = this.getmQClientFactory().getClientId(); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_CORRELATION_ID, correlationId); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, requestClientId); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_TTL, String.valueOf(timeout)); + + boolean hasRouteData = this.getmQClientFactory().getTopicRouteTable().containsKey(msg.getTopic()); + if (!hasRouteData) { + long beginTimestamp = System.currentTimeMillis(); + this.tryToFindTopicPublishInfo(msg.getTopic()); + this.getmQClientFactory().sendHeartbeatToAllBrokerWithLock(); + long cost = System.currentTimeMillis() - beginTimestamp; + if (cost > 500) { + log.warn("prepare send request for <{}> cost {} ms", msg.getTopic(), cost); + } + } + } + public ConcurrentMap getTopicPublishInfoTable() { return topicPublishInfoTable; } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index b4acf8f1c3241ce96a6c0729a00937bf97c54653..045734103f571e798508e4dfab3eaac738586c5e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -24,6 +24,7 @@ import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; @@ -42,38 +43,29 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; /** - * This class is the entry point for applications intending to send messages. - *

+ * This class is the entry point for applications intending to send messages.

* * It's fine to tune fields which exposes getter/setter methods, but keep in mind, all of them should work well out of - * box for most scenarios. - *

+ * box for most scenarios.

* * This class aggregates various send methods to deliver messages to brokers. Each of them has pros and - * cons; you'd better understand strengths and weakness of them before actually coding. - *

+ * cons; you'd better understand strengths and weakness of them before actually coding.

* - *

- * Thread Safety: After configuring and starting process, this class can be regarded as thread-safe - * and used among multiple threads context. - *

+ *

Thread Safety: After configuring and starting process, this class can be regarded as thread-safe + * and used among multiple threads context.

*/ public class DefaultMQProducer extends ClientConfig implements MQProducer { - private final InternalLogger log = ClientLogger.getLog(); - /** * Wrapping internal implementations for virtually all methods presented in this class. */ protected final transient DefaultMQProducerImpl defaultMQProducerImpl; - + private final InternalLogger log = ClientLogger.getLog(); /** * Producer group conceptually aggregates all producer instances of exactly same role, which is particularly - * important when transactional messages are involved. - *

+ * important when transactional messages are involved.

* - * For non-transactional messages, it does not matter as long as it's unique per process. - *

+ * For non-transactional messages, it does not matter as long as it's unique per process.

* * See {@linktourl http://rocketmq.apache.org/docs/core-concept/} for more discussion. */ @@ -100,16 +92,14 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { private int compressMsgBodyOverHowmuch = 1024 * 4; /** - * Maximum number of retry to perform internally before claiming sending failure in synchronous mode. - *

+ * Maximum number of retry to perform internally before claiming sending failure in synchronous mode.

* * This may potentially cause message duplication which is up to application developers to resolve. */ private int retryTimesWhenSendFailed = 2; /** - * Maximum number of retry to perform internally before claiming sending failure in asynchronous mode. - *

+ * Maximum number of retry to perform internally before claiming sending failure in asynchronous mode.

* * This may potentially cause message duplication which is up to application developers to resolve. */ @@ -171,7 +161,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { //if client open the message trace feature if (enableMsgTrace) { try { - AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(customizedTraceTopic, rpcHook); + AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(producerGroup, TraceDispatcher.Type.PRODUCE, customizedTraceTopic, rpcHook); dispatcher.setHostProducer(this.defaultMQProducerImpl); traceDispatcher = dispatcher; this.defaultMQProducerImpl.registerSendMessageHook( @@ -256,7 +246,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { //if client open the message trace feature if (enableMsgTrace) { try { - AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(customizedTraceTopic, rpcHook); + AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(producerGroup, TraceDispatcher.Type.PRODUCE, customizedTraceTopic, rpcHook); dispatcher.setHostProducer(this.getDefaultMQProducerImpl()); traceDispatcher = dispatcher; this.getDefaultMQProducerImpl().registerSendMessageHook( @@ -268,14 +258,10 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { } /** - * Start this producer instance. - *

+ * Start this producer instance.

* - * - * Much internal initializing procedures are carried out to make this instance prepared, thus, it's a must to invoke - * this method before sending or querying messages. - * - *

+ * Much internal initializing procedures are carried out to make this instance prepared, thus, it's a must + * to invoke this method before sending or querying messages.

* * @throws MQClientException if there is any unexpected error. */ @@ -316,8 +302,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { } /** - * Send message in synchronous mode. This method returns only when the sending procedure totally completes. - *

+ * Send message in synchronous mode. This method returns only when the sending procedure totally completes.

* * Warn: this method has internal retry-mechanism, that is, internal implementation will retry * {@link #retryTimesWhenSendFailed} times before claiming failure. As a result, multiple messages may potentially @@ -359,11 +344,9 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { } /** - * Send message to broker asynchronously. - *

+ * Send message to broker asynchronously.

* - * This method returns immediately. On sending completion, sendCallback will be executed. - *

+ * This method returns immediately. On sending completion, sendCallback will be executed.

* * Similar to {@link #send(Message)}, internal implementation would potentially retry up to {@link * #retryTimesWhenSendAsyncFailed} times before claiming sending failure, which may yield message duplication and @@ -582,6 +565,133 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { this.defaultMQProducerImpl.send(msg, selector, arg, sendCallback, timeout); } + /** + * Send request message in synchronous mode. This method returns only when the consumer consume the request message and reply a message.

+ * + * Warn: this method has internal retry-mechanism, that is, internal implementation will retry + * {@link #retryTimesWhenSendFailed} times before claiming failure. As a result, multiple messages may potentially + * delivered to broker(s). It's up to the application developers to resolve potential duplication issue. + * + * @param msg request message to send + * @param timeout request timeout + * @return reply message + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws MQBrokerException if there is any broker error. + * @throws InterruptedException if the thread is interrupted. + * @throws RequestTimeoutException if request timeout. + */ + @Override + public Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException, + RemotingException, MQBrokerException, InterruptedException { + msg.setTopic(withNamespace(msg.getTopic())); + return this.defaultMQProducerImpl.request(msg, timeout); + } + + /** + * Request asynchronously.

+ * This method returns immediately. On receiving reply message, requestCallback will be executed.

+ * + * Similar to {@link #request(Message, long)}, internal implementation would potentially retry up to {@link + * #retryTimesWhenSendAsyncFailed} times before claiming sending failure, which may yield message duplication and + * application developers are the one to resolve this potential issue. + * + * @param msg request message to send + * @param requestCallback callback to execute on request completion. + * @param timeout request timeout + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws InterruptedException if the thread is interrupted. + * @throws MQBrokerException if there is any broker error. + */ + @Override + public void request(final Message msg, final RequestCallback requestCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException, MQBrokerException { + msg.setTopic(withNamespace(msg.getTopic())); + this.defaultMQProducerImpl.request(msg, requestCallback, timeout); + } + + /** + * Same to {@link #request(Message, long)} with message queue selector specified. + * + * @param msg request message to send + * @param selector message queue selector, through which we get target message queue to deliver message to. + * @param arg argument to work along with message queue selector. + * @param timeout timeout of request. + * @return reply message + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws MQBrokerException if there is any broker error. + * @throws InterruptedException if the thread is interrupted. + * @throws RequestTimeoutException if request timeout. + */ + @Override + public Message request(final Message msg, final MessageQueueSelector selector, final Object arg, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException, RequestTimeoutException { + msg.setTopic(withNamespace(msg.getTopic())); + return this.defaultMQProducerImpl.request(msg, selector, arg, timeout); + } + + /** + * Same to {@link #request(Message, RequestCallback, long)} with target message selector specified. + * + * @param msg requst message to send + * @param selector message queue selector, through which we get target message queue to deliver message to. + * @param arg argument to work along with message queue selector. + * @param requestCallback callback to execute on request completion. + * @param timeout timeout of request. + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws InterruptedException if the thread is interrupted. + * @throws MQBrokerException if there is any broker error. + */ + @Override + public void request(final Message msg, final MessageQueueSelector selector, final Object arg, + final RequestCallback requestCallback, final long timeout) throws MQClientException, RemotingException, + InterruptedException, MQBrokerException { + msg.setTopic(withNamespace(msg.getTopic())); + this.defaultMQProducerImpl.request(msg, selector, arg, requestCallback, timeout); + } + + /** + * Same to {@link #request(Message, long)} with target message queue specified in addition. + * + * @param msg request message to send + * @param mq target message queue. + * @param timeout request timeout + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws MQBrokerException if there is any broker error. + * @throws InterruptedException if the thread is interrupted. + * @throws RequestTimeoutException if request timeout. + */ + @Override + public Message request(final Message msg, final MessageQueue mq, final long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { + msg.setTopic(withNamespace(msg.getTopic())); + return this.defaultMQProducerImpl.request(msg, mq, timeout); + } + + /** + * Same to {@link #request(Message, RequestCallback, long)} with target message queue specified. + * + * @param msg request message to send + * @param mq target message queue. + * @param requestCallback callback to execute on request completion. + * @param timeout timeout of request. + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws InterruptedException if the thread is interrupted. + * @throws MQBrokerException if there is any broker error. + */ + @Override + public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException, MQBrokerException { + msg.setTopic(withNamespace(msg.getTopic())); + this.defaultMQProducerImpl.request(msg, mq, requestCallback, timeout); + } + /** * Same to {@link #sendOneway(Message)} with message queue selector specified. * diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java index 1af6005748992d2334e0833c415500a9fbe96a57..c6cf4c93596578e958de17815fa9604e2c09b7fe 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.rocketmq.client.MQAdmin; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -98,4 +99,26 @@ public interface MQProducer extends MQAdmin { SendResult send(final Collection msgs, final MessageQueue mq, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException; + + //for rpc + Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException, + RemotingException, MQBrokerException, InterruptedException; + + void request(final Message msg, final RequestCallback requestCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException, MQBrokerException; + + Message request(final Message msg, final MessageQueueSelector selector, final Object arg, + final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, + InterruptedException; + + void request(final Message msg, final MessageQueueSelector selector, final Object arg, + final RequestCallback requestCallback, + final long timeout) throws MQClientException, RemotingException, + InterruptedException, MQBrokerException; + + Message request(final Message msg, final MessageQueue mq, final long timeout) + throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException; + + void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException; } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..3107ba57d6eaebeb6c982ba720facb302d0f24b6 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.producer; + +import org.apache.rocketmq.common.message.Message; + +public interface RequestCallback { + void onSuccess(final Message message); + + void onException(final Throwable e); +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java new file mode 100644 index 0000000000000000000000000000000000000000..3d4caa208bd203c8842c389f763181f20f36a0b0 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.producer; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.client.common.ClientErrorCode; +import org.apache.rocketmq.client.exception.RequestTimeoutException; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; + +public class RequestFutureTable { + private static InternalLogger log = ClientLogger.getLog(); + private static ConcurrentHashMap requestFutureTable = new ConcurrentHashMap(); + + public static ConcurrentHashMap getRequestFutureTable() { + return requestFutureTable; + } + + public static void scanExpiredRequest() { + final List rfList = new LinkedList(); + Iterator> it = requestFutureTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + RequestResponseFuture rep = next.getValue(); + + if (rep.isTimeout()) { + it.remove(); + rfList.add(rep); + log.warn("remove timeout request, CorrelationId={}" + rep.getCorrelationId()); + } + } + + for (RequestResponseFuture rf : rfList) { + try { + Throwable cause = new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, "request timeout, no reply message."); + rf.setCause(cause); + rf.executeRequestCallback(); + } catch (Throwable e) { + log.warn("scanResponseTable, operationComplete Exception", e); + } + } + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java new file mode 100644 index 0000000000000000000000000000000000000000..c54b236b1616fcf9d4f586af8ac2f1e52d09c85f --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.producer; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.common.message.Message; + +public class RequestResponseFuture { + private final String correlationId; + private final RequestCallback requestCallback; + private final long beginTimestamp = System.currentTimeMillis(); + private final Message requestMsg = null; + private long timeoutMillis; + private CountDownLatch countDownLatch = new CountDownLatch(1); + private volatile Message responseMsg = null; + private volatile boolean sendRequestOk = true; + private volatile Throwable cause = null; + + public RequestResponseFuture(String correlationId, long timeoutMillis, RequestCallback requestCallback) { + this.correlationId = correlationId; + this.timeoutMillis = timeoutMillis; + this.requestCallback = requestCallback; + } + + public void executeRequestCallback() { + if (requestCallback != null) { + if (sendRequestOk && cause == null) { + requestCallback.onSuccess(responseMsg); + } else { + requestCallback.onException(cause); + } + } + } + + public boolean isTimeout() { + long diff = System.currentTimeMillis() - this.beginTimestamp; + return diff > this.timeoutMillis; + } + + public Message waitResponseMessage(final long timeout) throws InterruptedException { + this.countDownLatch.await(timeout, TimeUnit.MILLISECONDS); + return this.responseMsg; + } + + public void putResponseMessage(final Message responseMsg) { + this.responseMsg = responseMsg; + this.countDownLatch.countDown(); + } + + public String getCorrelationId() { + return correlationId; + } + + public long getTimeoutMillis() { + return timeoutMillis; + } + + public void setTimeoutMillis(long timeoutMillis) { + this.timeoutMillis = timeoutMillis; + } + + public RequestCallback getRequestCallback() { + return requestCallback; + } + + public long getBeginTimestamp() { + return beginTimestamp; + } + + public CountDownLatch getCountDownLatch() { + return countDownLatch; + } + + public void setCountDownLatch(CountDownLatch countDownLatch) { + this.countDownLatch = countDownLatch; + } + + public Message getResponseMsg() { + return responseMsg; + } + + public void setResponseMsg(Message responseMsg) { + this.responseMsg = responseMsg; + } + + public boolean isSendRequestOk() { + return sendRequestOk; + } + + public void setSendReqeustOk(boolean sendReqeustOk) { + this.sendRequestOk = sendReqeustOk; + } + + public Message getRequestMsg() { + return requestMsg; + } + + public Throwable getCause() { + return cause; + } + + public void setCause(Throwable cause) { + this.cause = cause; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java index ca3bcfa261a574cd6df668d99f4b8170d4eb40da..8c3d886517595ce2fb53d1012735d8fb7af32b50 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java @@ -73,14 +73,19 @@ public class AsyncTraceDispatcher implements TraceDispatcher { private String traceTopicName; private AtomicBoolean isStarted = new AtomicBoolean(false); private AccessChannel accessChannel = AccessChannel.LOCAL; + private String group; + private Type type; - public AsyncTraceDispatcher(String traceTopicName, RPCHook rpcHook) { + public AsyncTraceDispatcher(String group, Type type,String traceTopicName, RPCHook rpcHook) { // queueSize is greater than or equal to the n power of 2 of value this.queueSize = 2048; this.batchSize = 100; this.maxMsgSize = 128000; this.discardCount = new AtomicLong(0L); this.traceContextQueue = new ArrayBlockingQueue(1024); + this.group = group; + this.type = type; + this.appenderQueue = new ArrayBlockingQueue(queueSize); if (!UtilAll.isBlank(traceTopicName)) { this.traceTopicName = traceTopicName; @@ -150,7 +155,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher { DefaultMQProducer traceProducerInstance = this.traceProducer; if (traceProducerInstance == null) { traceProducerInstance = new DefaultMQProducer(rpcHook); - traceProducerInstance.setProducerGroup(TraceConstants.GROUP_NAME); + traceProducerInstance.setProducerGroup(genGroupNameForTrace()); traceProducerInstance.setSendMsgTimeout(5000); traceProducerInstance.setVipChannelEnabled(false); // The max size of message is 128K @@ -159,6 +164,10 @@ public class AsyncTraceDispatcher implements TraceDispatcher { return traceProducerInstance; } + private String genGroupNameForTrace() { + return TraceConstants.GROUP_NAME_PREFIX + "-" + this.group + "-" + this.type ; + } + @Override public boolean append(final Object ctx) { boolean result = traceContextQueue.offer((TraceContext) ctx); @@ -216,7 +225,11 @@ public class AsyncTraceDispatcher implements TraceDispatcher { public void removeShutdownHook() { if (shutDownHook != null) { - Runtime.getRuntime().removeShutdownHook(shutDownHook); + try { + Runtime.getRuntime().removeShutdownHook(shutDownHook); + } catch (IllegalStateException e) { + // ignore - VM is already shutting down + } } } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java index e61ea9d1a0591cf8cdcf6e9688ad73b498b4d61f..cb4a246517d62e711bd7b7dd171fb4e97d575482 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java @@ -20,7 +20,7 @@ import org.apache.rocketmq.common.MixAll; public class TraceConstants { - public static final String GROUP_NAME = "_INNER_TRACE_PRODUCER"; + public static final String GROUP_NAME_PREFIX = "_INNER_TRACE_PRODUCER"; public static final char CONTENT_SPLITOR = (char) 1; public static final char FIELD_SPLITOR = (char) 2; public static final String TRACE_INSTANCE_NAME = "PID_CLIENT_INNER_TRACE_PRODUCER"; diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcher.java index 51cc0deb87d61fc81552cd8b04b8080945ce6042..33341cf6b8f8b6e5eb7be2c9c4fddd6fe22f7586 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcher.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDispatcher.java @@ -24,7 +24,10 @@ import java.io.IOException; * Interface of asynchronous transfer data */ public interface TraceDispatcher { - + enum Type { + PRODUCE, + CONSUME + } /** * Initialize asynchronous transfer data module */ diff --git a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..416ba44da3284eaa711a10a735b8b6edeb1e51f8 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.utils; + +import org.apache.rocketmq.client.common.ClientErrorCode; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; + +public class MessageUtil { + public static Message createReplyMessage(final Message requestMessage, final byte[] body) throws MQClientException { + if (requestMessage != null) { + Message replyMessage = new Message(); + String cluster = requestMessage.getProperty(MessageConst.PROPERTY_CLUSTER); + String replyTo = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT); + String correlationId = requestMessage.getProperty(MessageConst.PROPERTY_CORRELATION_ID); + String ttl = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_TTL); + replyMessage.setBody(body); + if (cluster != null) { + String replyTopic = MixAll.getReplyTopic(cluster); + replyMessage.setTopic(replyTopic); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TYPE, MixAll.REPLY_MESSAGE_FLAG); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_CORRELATION_ID, correlationId); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, replyTo); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TTL, ttl); + + return replyMessage; + } else { + throw new MQClientException(ClientErrorCode.CREATE_REPLY_MESSAGE_EXCEPTION, "create reply message fail, requestMessage error, property[" + MessageConst.PROPERTY_CLUSTER + "] is null."); + } + } + throw new MQClientException(ClientErrorCode.CREATE_REPLY_MESSAGE_EXCEPTION, "create reply message fail, requestMessage cannot be null."); + } + + public static String getReplyToClient(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT); + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java b/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java index 5a012ceb087a008308b60c3c42a81f0c2d0a6e52..e2b9abd46f045bdefbc4f78bdc8112ac899b4563 100644 --- a/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java @@ -71,13 +71,13 @@ public class ValidatorsTest { @Test public void testCheckTopic_TooLongTopic() { - String tooLongTopic = StringUtils.rightPad("TooLongTopic", Validators.CHARACTER_MAX_LENGTH + 1, "_"); - assertThat(tooLongTopic.length()).isGreaterThan(Validators.CHARACTER_MAX_LENGTH); + String tooLongTopic = StringUtils.rightPad("TooLongTopic", Validators.TOPIC_MAX_LENGTH + 1, "_"); + assertThat(tooLongTopic.length()).isGreaterThan(Validators.TOPIC_MAX_LENGTH); try { Validators.checkTopic(tooLongTopic); failBecauseExceptionWasNotThrown(MQClientException.class); } catch (MQClientException e) { - assertThat(e).hasMessageStartingWith("The specified topic is longer than topic max length 255."); + assertThat(e).hasMessageStartingWith("The specified topic is longer than topic max length"); } } } diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java new file mode 100644 index 0000000000000000000000000000000000000000..cc8d5e2bf78a8ba0c840c1ec4e1045b89f931009 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java @@ -0,0 +1,575 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.consumer; + +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.consumer.store.ReadOffsetType; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQAdminImpl; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.consumer.AssignedMessageQueue; +import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; +import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.impl.consumer.RebalanceImpl; +import org.apache.rocketmq.client.impl.consumer.RebalanceService; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageClientExt; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(DefaultLitePullConsumerImpl.class) +public class DefaultLitePullConsumerTest { + @Spy + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); + + @Mock + private MQClientAPIImpl mQClientAPIImpl; + @Mock + private MQAdminImpl mQAdminImpl; + + private RebalanceImpl rebalanceImpl; + private OffsetStore offsetStore; + private DefaultLitePullConsumerImpl litePullConsumerImpl; + private String consumerGroup = "LitePullConsumerGroup"; + private String topic = "LitePullConsumerTest"; + private String brokerName = "BrokerA"; + private boolean flag = false; + + @Before + public void init() throws Exception { + PowerMockito.suppress(PowerMockito.method(DefaultLitePullConsumerImpl.class, "updateTopicSubscribeInfoWhenSubscriptionChanged")); + Field field = MQClientInstance.class.getDeclaredField("rebalanceService"); + field.setAccessible(true); + RebalanceService rebalanceService = (RebalanceService) field.get(mQClientFactory); + field = RebalanceService.class.getDeclaredField("waitInterval"); + field.setAccessible(true); + field.set(rebalanceService, 100); + } + + @Test + public void testAssign_PollMessageSuccess() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testSubscribe_PollMessageSuccess() throws Exception { + DefaultLitePullConsumer litePullConsumer = createSubscribeLitePullConsumer(); + try { + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testSubscribe_BroadcastPollMessageSuccess() throws Exception { + DefaultLitePullConsumer litePullConsumer = createBroadcastLitePullConsumer(); + try { + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testSubscriptionType_AssignAndSubscribeExclusive() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + try { + litePullConsumer.subscribe(topic, "*"); + litePullConsumer.assign(Collections.singletonList(createMessageQueue())); + failBecauseExceptionWasNotThrown(IllegalStateException.class); + } catch (IllegalStateException e) { + assertThat(e).hasMessageContaining("Subscribe and assign are mutually exclusive."); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testFetchMessageQueues_FetchMessageQueuesBeforeStart() throws Exception { + DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer(); + try { + litePullConsumer.fetchMessageQueues(topic); + failBecauseExceptionWasNotThrown(IllegalStateException.class); + } catch (IllegalStateException e) { + assertThat(e).hasMessageContaining("The consumer not running, please start it first."); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testSeek_SeekOffsetSuccess() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); + when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L); + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + long offset = litePullConsumer.committed(messageQueue); + litePullConsumer.seek(messageQueue, offset); + Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); + field.setAccessible(true); + AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl); + assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), offset); + litePullConsumer.shutdown(); + } + + @Test + public void testSeek_SeekToBegin() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); + when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L); + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.seekToBegin(messageQueue); + Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); + field.setAccessible(true); + AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl); + assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), 0L); + litePullConsumer.shutdown(); + } + + @Test + public void testSeek_SeekToEnd() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); + when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L); + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.seekToEnd(messageQueue); + Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); + field.setAccessible(true); + AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl); + assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), 500L); + litePullConsumer.shutdown(); + } + + @Test + public void testSeek_SeekOffsetIllegal() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); + when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(100L); + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + try { + litePullConsumer.seek(messageQueue, -1); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("min offset = 0"); + } + + try { + litePullConsumer.seek(messageQueue, 1000); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("max offset = 100"); + } + litePullConsumer.shutdown(); + } + + @Test + public void testSeek_MessageQueueNotInAssignList() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + try { + litePullConsumer.seek(createMessageQueue(), 0); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("The message queue is not in assigned list"); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = createSubscribeLitePullConsumer(); + try { + litePullConsumer.seek(createMessageQueue(), 0); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("The message queue is not in assigned list, may be rebalancing"); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testOffsetForTimestamp_FailedAndSuccess() throws Exception { + MessageQueue messageQueue = createMessageQueue(); + DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer(); + try { + litePullConsumer.offsetForTimestamp(messageQueue, 123456L); + failBecauseExceptionWasNotThrown(IllegalStateException.class); + } catch (IllegalStateException e) { + assertThat(e).hasMessageContaining("The consumer not running, please start it first."); + } finally { + litePullConsumer.shutdown(); + } + doReturn(123L).when(mQAdminImpl).searchOffset(any(MessageQueue.class), anyLong()); + litePullConsumer = createStartLitePullConsumer(); + long offset = litePullConsumer.offsetForTimestamp(messageQueue, 123456L); + assertThat(offset).isEqualTo(123L); + } + + @Test + public void testPauseAndResume_Success() throws Exception { + DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.pause(Collections.singletonList(messageQueue)); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + List result = litePullConsumer.poll(); + assertThat(result.isEmpty()).isTrue(); + litePullConsumer.resume(Collections.singletonList(messageQueue)); + result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testRegisterTopicMessageQueueChangeListener_Success() throws Exception { + flag = false; + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + doReturn(Collections.emptySet()).when(mQAdminImpl).fetchSubscribeMessageQueues(anyString()); + litePullConsumer.setTopicMetadataCheckIntervalMillis(10); + litePullConsumer.registerTopicMessageQueueChangeListener(topic, new TopicMessageQueueChangeListener() { + @Override public void onChanged(String topic, Set messageQueues) { + flag = true; + } + }); + Set set = new HashSet(); + set.add(createMessageQueue()); + doReturn(set).when(mQAdminImpl).fetchSubscribeMessageQueues(anyString()); + Thread.sleep(11 * 1000); + assertThat(flag).isTrue(); + } + + @Test + public void testFlowControl_Success() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.setPullThresholdForAll(-1); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.setPollTimeoutMillis(500); + List result = litePullConsumer.poll(); + assertThat(result).isEmpty(); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = createStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.setPullThresholdForQueue(-1); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.setPollTimeoutMillis(500); + List result = litePullConsumer.poll(); + assertThat(result).isEmpty(); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = createStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.setPullThresholdSizeForQueue(-1); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.setPollTimeoutMillis(500); + List result = litePullConsumer.poll(); + assertThat(result).isEmpty(); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = createStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.setConsumeMaxSpan(-1); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.setPollTimeoutMillis(500); + List result = litePullConsumer.poll(); + assertThat(result).isEmpty(); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testCheckConfig_Exception() { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(MixAll.DEFAULT_CONSUMER_GROUP); + try { + litePullConsumer.start(); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("consumerGroup can not equal"); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setMessageModel(null); + try { + litePullConsumer.start(); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("messageModel is null"); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setAllocateMessageQueueStrategy(null); + try { + litePullConsumer.start(); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("allocateMessageQueueStrategy is null"); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setConsumerTimeoutMillisWhenSuspend(1); + try { + litePullConsumer.start(); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("Long polling mode, the consumer consumerTimeoutMillisWhenSuspend must greater than brokerSuspendMaxTimeMillis"); + } finally { + litePullConsumer.shutdown(); + } + + } + + @Test + public void testComputePullFromWhereReturnedNotFound() throws Exception{ + DefaultLitePullConsumer defaultLitePullConsumer = createStartLitePullConsumer(); + defaultLitePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + MessageQueue messageQueue = createMessageQueue(); + when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L); + long offset = rebalanceImpl.computePullFromWhere(messageQueue); + assertThat(offset).isEqualTo(0); + } + + @Test + public void testComputePullFromWhereReturned() throws Exception{ + DefaultLitePullConsumer defaultLitePullConsumer = createStartLitePullConsumer(); + defaultLitePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + MessageQueue messageQueue = createMessageQueue(); + when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(100L); + long offset = rebalanceImpl.computePullFromWhere(messageQueue); + assertThat(offset).isEqualTo(100); + } + + + @Test + public void testComputePullFromLast() throws Exception{ + DefaultLitePullConsumer defaultLitePullConsumer = createStartLitePullConsumer(); + defaultLitePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + MessageQueue messageQueue = createMessageQueue(); + when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L); + when(mQClientFactory.getMQAdminImpl().maxOffset(any(MessageQueue.class))).thenReturn(100L); + long offset = rebalanceImpl.computePullFromWhere(messageQueue); + assertThat(offset).isEqualTo(100); + } + + @Test + public void testComputePullByTimeStamp() throws Exception{ + DefaultLitePullConsumer defaultLitePullConsumer = createStartLitePullConsumer(); + defaultLitePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP); + defaultLitePullConsumer.setConsumeTimestamp("20191024171201"); + MessageQueue messageQueue = createMessageQueue(); + when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(-1L); + when(mQClientFactory.getMQAdminImpl().searchOffset(any(MessageQueue.class),anyLong())).thenReturn(100L); + long offset = rebalanceImpl.computePullFromWhere(messageQueue); + assertThat(offset).isEqualTo(100); + } + + private void initDefaultLitePullConsumer(DefaultLitePullConsumer litePullConsumer) throws Exception { + + Field field = DefaultLitePullConsumer.class.getDeclaredField("defaultLitePullConsumerImpl"); + field.setAccessible(true); + litePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(litePullConsumerImpl, mQClientFactory); + + PullAPIWrapper pullAPIWrapper = litePullConsumerImpl.getPullAPIWrapper(); + field = PullAPIWrapper.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(pullAPIWrapper, mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + field = MQClientInstance.class.getDeclaredField("mQAdminImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQAdminImpl); + + field = DefaultLitePullConsumerImpl.class.getDeclaredField("rebalanceImpl"); + field.setAccessible(true); + rebalanceImpl = (RebalanceImpl) field.get(litePullConsumerImpl); + field = RebalanceImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(rebalanceImpl, mQClientFactory); + + offsetStore = spy(litePullConsumerImpl.getOffsetStore()); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("offsetStore"); + field.setAccessible(true); + field.set(litePullConsumerImpl, offsetStore); + + when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), + anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) + .thenAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + MessageClientExt messageClientExt = new MessageClientExt(); + messageClientExt.setTopic(topic); + messageClientExt.setQueueId(0); + messageClientExt.setMsgId("123"); + messageClientExt.setBody(new byte[] {'a'}); + messageClientExt.setOffsetMsgId("234"); + messageClientExt.setBornHost(new InetSocketAddress(8080)); + messageClientExt.setStoreHost(new InetSocketAddress(8080)); + PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); + return pullResult; + } + }); + + when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(new FindBrokerResult("127.0.0.1:10911", false)); + + doReturn(Collections.singletonList(mQClientFactory.getClientId())).when(mQClientFactory).findConsumerIdList(anyString(), anyString()); + + doReturn(123L).when(offsetStore).readOffset(any(MessageQueue.class), any(ReadOffsetType.class)); + } + + private DefaultLitePullConsumer createSubscribeLitePullConsumer() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.subscribe(topic, "*"); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; + } + + private DefaultLitePullConsumer createStartLitePullConsumer() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; + } + + private DefaultLitePullConsumer createNotStartLitePullConsumer() { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + return litePullConsumer; + } + + private DefaultLitePullConsumer createBroadcastLitePullConsumer() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.setMessageModel(MessageModel.BROADCASTING); + litePullConsumer.subscribe(topic, "*"); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; + } + + private MessageQueue createMessageQueue() { + MessageQueue messageQueue = new MessageQueue(); + messageQueue.setBrokerName(brokerName); + messageQueue.setQueueId(0); + messageQueue.setTopic(topic); + return messageQueue; + } + + private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, + List messageExtList) throws Exception { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (MessageExt messageExt : messageExtList) { + outputStream.write(MessageDecoder.encode(messageExt, false)); + } + return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerTest.java index 569055dea2566068b0ca165d893b710dcd08b751..7afaf2bea05738779d6379effaa0d358c3c4a16a 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerTest.java @@ -54,7 +54,7 @@ import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class DefaultMQPullConsumerTest { @Spy - private MQClientInstance mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); @Mock private MQClientAPIImpl mQClientAPIImpl; private DefaultMQPullConsumer pullConsumer; diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index ff2fb78bbfb9df89596abc2ad7b35fcc33a2df3e..e6f0e866882ec460ba4505d007fca8e0ebfd995c 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -59,8 +59,10 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; -import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; @@ -73,7 +75,8 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest(DefaultMQPushConsumerImpl.class) public class DefaultMQPushConsumerTest { private String consumerGroup; private String topic = "FooBar"; @@ -102,10 +105,12 @@ public class DefaultMQPushConsumerTest { }); DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl(); + PowerMockito.suppress(PowerMockito.method(DefaultMQPushConsumerImpl.class, "updateTopicSubscribeInfoWhenSubscriptionChanged")); rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl())); Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); field.setAccessible(true); field.set(pushConsumerImpl, rebalancePushImpl); + pushConsumer.subscribe(topic, "*"); pushConsumer.start(); diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java index 84af63235570bcfbe9d9919a7b39cfe0283e91a5..3f00d9e4030473f395c9b2a037e50b7e6fde24f0 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java @@ -42,6 +42,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Matchers; import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; @@ -164,7 +165,7 @@ public class MQClientAPIImplTest { public Object answer(InvocationOnMock mock) throws Throwable { InvokeCallback callback = mock.getArgument(3); RemotingCommand request = mock.getArgument(1); - ResponseFuture responseFuture = new ResponseFuture(null,request.getOpaque(), 3 * 1000, null, null); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); responseFuture.setResponseCommand(createSuccessResponse(request)); callback.operationComplete(responseFuture); return null; @@ -289,6 +290,7 @@ public class MQClientAPIImplTest { assertThat(ex.getErrorMessage()).isEqualTo("corresponding to accessConfig has been deleted failed"); } } + @Test public void testResumeCheckHalfMessage_WithException() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { doAnswer(new Answer() { @@ -322,6 +324,38 @@ public class MQClientAPIImplTest { assertThat(result).isEqualTo(true); } + @Test + public void testSendMessageTypeofReply() throws Exception { + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock mock) throws Throwable { + InvokeCallback callback = mock.getArgument(3); + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + responseFuture.setResponseCommand(createSuccessResponse(request)); + callback.operationComplete(responseFuture); + return null; + } + }).when(remotingClient).invokeAsync(Matchers.anyString(), Matchers.any(RemotingCommand.class), Matchers.anyLong(), Matchers.any(InvokeCallback.class)); + SendMessageContext sendMessageContext = new SendMessageContext(); + sendMessageContext.setProducer(new DefaultMQProducerImpl(new DefaultMQProducer())); + msg.getProperties().put("MSG_TYPE", "reply"); + mqClientAPI.sendMessage(brokerAddr, brokerName, msg, new SendMessageRequestHeader(), 3 * 1000, CommunicationMode.ASYNC, + new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); + assertThat(sendResult.getQueueOffset()).isEqualTo(123L); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(1); + } + + @Override + public void onException(Throwable e) { + } + }, null, null, 0, sendMessageContext, defaultMQProducerImpl); + } + private RemotingCommand createResumeSuccessResponse(RemotingCommand request) { RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setCode(ResponseCode.SUCCESS); diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyServiceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..4cfa011434c962f3affa705ed4dbd9ea691b1c10 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyServiceTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.body.CMResult; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class ConsumeMessageOrderlyServiceTest { + private String consumerGroup; + private String topic = "FooBar"; + private String brokerName = "BrokerA"; + private DefaultMQPushConsumer pushConsumer; + + @Before + public void init() throws Exception { + consumerGroup = "FooBarGroup" + System.currentTimeMillis(); + pushConsumer = new DefaultMQPushConsumer(consumerGroup); + } + + @Test + public void testConsumeMessageDirectly_WithNoException() { + Map map = new HashMap(); + map.put(ConsumeOrderlyStatus.SUCCESS, CMResult.CR_SUCCESS); + map.put(ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT, CMResult.CR_LATER); + map.put(ConsumeOrderlyStatus.COMMIT, CMResult.CR_COMMIT); + map.put(ConsumeOrderlyStatus.ROLLBACK, CMResult.CR_ROLLBACK); + map.put(null, CMResult.CR_RETURN_NULL); + + for (ConsumeOrderlyStatus consumeOrderlyStatus : map.keySet()) { + final ConsumeOrderlyStatus status = consumeOrderlyStatus; + MessageListenerOrderly listenerOrderly = new MessageListenerOrderly() { + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, ConsumeOrderlyContext context) { + return status; + } + }; + + ConsumeMessageOrderlyService consumeMessageOrderlyService = new ConsumeMessageOrderlyService(pushConsumer.getDefaultMQPushConsumerImpl(), listenerOrderly); + MessageExt msg = new MessageExt(); + msg.setTopic(topic); + assertTrue(consumeMessageOrderlyService.consumeMessageDirectly(msg, brokerName).getConsumeResult().equals(map.get(consumeOrderlyStatus))); + } + + } + + @Test + public void testConsumeMessageDirectly_WithException() { + MessageListenerOrderly listenerOrderly = new MessageListenerOrderly() { + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, ConsumeOrderlyContext context) { + throw new RuntimeException(); + } + }; + + ConsumeMessageOrderlyService consumeMessageOrderlyService = new ConsumeMessageOrderlyService(pushConsumer.getDefaultMQPushConsumerImpl(), listenerOrderly); + MessageExt msg = new MessageExt(); + msg.setTopic(topic); + assertTrue(consumeMessageOrderlyService.consumeMessageDirectly(msg, brokerName).getConsumeResult().equals(CMResult.CR_THROW_EXCEPTION)); + } + +} diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java index 171a95a8b68f520c1c9e833ad85fc66880b2cd35..bb2132111c179e8fb13dc283aa4634f2313e6040 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java @@ -39,7 +39,7 @@ import static org.mockito.Mockito.mock; @RunWith(MockitoJUnitRunner.class) public class MQClientInstanceTest { - private MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private String topic = "FooBar"; private String group = "FooBarGroup"; diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 9540755fe339f37651f983fbe1ab524e7fa94004..d572a2320e30eecbb91bf74b4e80a784a971937d 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -21,15 +21,18 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.hook.SendMessageContext; import org.apache.rocketmq.client.hook.SendMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; @@ -45,6 +48,7 @@ import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.junit.After; import org.junit.Before; @@ -52,7 +56,9 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; @@ -66,7 +72,7 @@ import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class DefaultMQProducerTest { @Spy - private MQClientInstance mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); @Mock private MQClientAPIImpl mQClientAPIImpl; @Mock @@ -168,6 +174,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(message, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -184,14 +191,17 @@ public class DefaultMQProducerTest { }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); } + @Test public void testSendMessageAsync() throws RemotingException, MQClientException, InterruptedException { final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(6); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); SendCallback sendCallback = new SendCallback() { @Override public void onSuccess(SendResult sendResult) { + countDownLatch.countDown(); } @Override @@ -211,21 +221,22 @@ public class DefaultMQProducerTest { Message message = new Message(); message.setTopic("test"); message.setBody("hello world".getBytes()); - producer.send(new Message(),sendCallback); - producer.send(message,sendCallback,1000); - producer.send(message,new MessageQueue(),sendCallback); - producer.send(new Message(),new MessageQueue(),sendCallback,1000); - producer.send(new Message(),messageQueueSelector,null,sendCallback); - producer.send(message,messageQueueSelector,null,sendCallback,1000); + producer.send(new Message(), sendCallback); + producer.send(message, new MessageQueue(), sendCallback); + producer.send(new Message(), new MessageQueue(), sendCallback, 1000); + producer.send(new Message(), messageQueueSelector, null, sendCallback); + producer.send(message, messageQueueSelector, null, sendCallback, 1000); + //this message is send success + producer.send(message, sendCallback, 1000); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); - assertThat(cc.get()).isEqualTo(6); + assertThat(cc.get()).isEqualTo(5); } @Test public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - final CountDownLatch countDownLatch = new CountDownLatch(1); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(bigMessage, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -237,7 +248,6 @@ public class DefaultMQProducerTest { @Override public void onException(Throwable e) { - countDownLatch.countDown(); } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); @@ -311,6 +321,101 @@ public class DefaultMQProducerTest { assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized); } + @Test + public void testRequestMessage() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + final AtomicBoolean finish = new AtomicBoolean(false); + new Thread(new Runnable() { + @Override public void run() { + ConcurrentHashMap responseMap = RequestFutureTable.getRequestFutureTable(); + assertThat(responseMap).isNotNull(); + while (!finish.get()) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + } + for (Map.Entry entry : responseMap.entrySet()) { + RequestResponseFuture future = entry.getValue(); + future.putResponseMessage(message); + } + } + } + }).start(); + Message result = producer.request(message, 3 * 1000L); + finish.getAndSet(true); + assertThat(result.getTopic()).isEqualTo("FooBar"); + assertThat(result.getBody()).isEqualTo(new byte[] {'a'}); + } + + @Test(expected = RequestTimeoutException.class) + public void testRequestMessage_RequestTimeoutException() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + Message result = producer.request(message, 3 * 1000L); + } + + @Test + public void testAsyncRequest_OnSuccess() throws Exception { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + final CountDownLatch countDownLatch = new CountDownLatch(1); + RequestCallback requestCallback = new RequestCallback() { + @Override public void onSuccess(Message message) { + assertThat(message.getTopic()).isEqualTo("FooBar"); + assertThat(message.getBody()).isEqualTo(new byte[] {'a'}); + assertThat(message.getFlag()).isEqualTo(1); + countDownLatch.countDown(); + } + + @Override public void onException(Throwable e) { + } + }; + producer.request(message, requestCallback, 3 * 1000L); + ConcurrentHashMap responseMap = RequestFutureTable.getRequestFutureTable(); + assertThat(responseMap).isNotNull(); + for (Map.Entry entry : responseMap.entrySet()) { + RequestResponseFuture future = entry.getValue(); + future.setSendReqeustOk(true); + message.setFlag(1); + future.getRequestCallback().onSuccess(message); + } + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + } + + @Test + public void testAsyncRequest_OnException() throws Exception { + final AtomicInteger cc = new AtomicInteger(0); + final CountDownLatch countDownLatch = new CountDownLatch(1); + RequestCallback requestCallback = new RequestCallback() { + @Override public void onSuccess(Message message) { + + } + + @Override public void onException(Throwable e) { + cc.incrementAndGet(); + countDownLatch.countDown(); + } + }; + MessageQueueSelector messageQueueSelector = new MessageQueueSelector() { + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + return null; + } + }; + + try { + producer.request(message, requestCallback, 3 * 1000L); + failBecauseExceptionWasNotThrown(Exception.class); + } catch (Exception e) { + ConcurrentHashMap responseMap = RequestFutureTable.getRequestFutureTable(); + assertThat(responseMap).isNotNull(); + for (Map.Entry entry : responseMap.entrySet()) { + RequestResponseFuture future = entry.getValue(); + future.getRequestCallback().onException(e); + } + } + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + assertThat(cc.get()).isEqualTo(1); + } + public static TopicRouteData createTopicRoute() { TopicRouteData topicRouteData = new TopicRouteData(); diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/RequestResponseFutureTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/RequestResponseFutureTest.java new file mode 100644 index 0000000000000000000000000000000000000000..90e4623e9b486770b1740d090a2dc9f05aca51f5 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/producer/RequestResponseFutureTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.producer; + +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.rocketmq.common.message.Message; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class RequestResponseFutureTest { + + @Test + public void testExecuteRequestCallback() throws Exception { + final AtomicInteger cc = new AtomicInteger(0); + RequestResponseFuture future = new RequestResponseFuture(UUID.randomUUID().toString(), 3 * 1000L, new RequestCallback() { + @Override public void onSuccess(Message message) { + cc.incrementAndGet(); + } + + @Override public void onException(Throwable e) { + } + }); + future.setSendReqeustOk(true); + future.executeRequestCallback(); + assertThat(cc.get()).isEqualTo(1); + } + +} diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java index 0d00c9bdb448841226c093e1cf0d00d532c62104..496c5143e02387f5a39289f8a77ce61021fbd927 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java @@ -70,8 +70,10 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; -import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -83,7 +85,8 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest(DefaultMQPushConsumerImpl.class) public class DefaultMQConsumerWithTraceTest { private String consumerGroup; private String consumerGroupNormal; @@ -101,7 +104,6 @@ public class DefaultMQConsumerWithTraceTest { private DefaultMQPushConsumer normalPushConsumer; private DefaultMQPushConsumer customTraceTopicpushConsumer; - private AsyncTraceDispatcher asyncTraceDispatcher; private MQClientInstance mQClientTraceFactory; @Mock @@ -112,17 +114,16 @@ public class DefaultMQConsumerWithTraceTest { @Before public void init() throws Exception { consumerGroup = "FooBarGroup" + System.currentTimeMillis(); - pushConsumer = new DefaultMQPushConsumer(consumerGroup,true,""); + pushConsumer = new DefaultMQPushConsumer(consumerGroup, true, ""); consumerGroupNormal = "FooBarGroup" + System.currentTimeMillis(); - normalPushConsumer = new DefaultMQPushConsumer(consumerGroupNormal,false,""); - customTraceTopicpushConsumer = new DefaultMQPushConsumer(consumerGroup,true,customerTraceTopic); + normalPushConsumer = new DefaultMQPushConsumer(consumerGroupNormal, false, ""); + customTraceTopicpushConsumer = new DefaultMQPushConsumer(consumerGroup, true, customerTraceTopic); pushConsumer.setNamesrvAddr("127.0.0.1:9876"); pushConsumer.setPullInterval(60 * 1000); - asyncTraceDispatcher = (AsyncTraceDispatcher)pushConsumer.getTraceDispatcher(); + asyncTraceDispatcher = (AsyncTraceDispatcher) pushConsumer.getTraceDispatcher(); traceProducer = asyncTraceDispatcher.getTraceProducer(); - pushConsumer.registerMessageListener(new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, @@ -131,12 +132,14 @@ public class DefaultMQConsumerWithTraceTest { } }); + PowerMockito.suppress(PowerMockito.method(DefaultMQPushConsumerImpl.class, "updateTopicSubscribeInfoWhenSubscriptionChanged")); DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl(); rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl())); Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); field.setAccessible(true); field.set(pushConsumerImpl, rebalancePushImpl); pushConsumer.subscribe(topic, "*"); + pushConsumer.start(); mQClientFactory = spy(pushConsumerImpl.getmQClientFactory()); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java index 903be01cddbf13031368db73daa9df8ab6198ebb..3759acba139f9d7eb3d343855acbc27e338512b7 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java @@ -60,7 +60,7 @@ import static org.mockito.Mockito.when; public class DefaultMQProducerWithTraceTest { @Spy - private MQClientInstance mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); @Mock private MQClientAPIImpl mQClientAPIImpl; @@ -87,7 +87,7 @@ public class DefaultMQProducerWithTraceTest { producer.setNamesrvAddr("127.0.0.1:9876"); normalProducer.setNamesrvAddr("127.0.0.1:9877"); customTraceTopicproducer.setNamesrvAddr("127.0.0.1:9878"); - message = new Message(topic, new byte[]{'a', 'b', 'c'}); + message = new Message(topic, new byte[] {'a', 'b', 'c'}); asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher(); asyncTraceDispatcher.setTraceTopicName(customerTraceTopic); asyncTraceDispatcher.getHostProducer(); @@ -108,14 +108,13 @@ public class DefaultMQProducerWithTraceTest { field.setAccessible(true); field.set(mQClientFactory, mQClientAPIImpl); - producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl()); when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), - nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); + nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), - nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenReturn(createSendResult(SendStatus.SEND_OK)); + nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(createSendResult(SendStatus.SEND_OK)); } diff --git a/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java b/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java new file mode 100644 index 0000000000000000000000000000000000000000..803e596fc829bc1cd666563f70bd35c7e824cc13 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.utils; + +import java.util.HashMap; +import java.util.Map; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; + +public class MessageUtilsTest { + + @Test + public void testCreateReplyMessage() throws MQClientException { + Message msg = MessageUtil.createReplyMessage(createReplyMessage("clusterName"), new byte[] {'a'}); + assertThat(msg.getTopic()).isEqualTo("clusterName" + "_" + MixAll.REPLY_TOPIC_POSTFIX); + assertThat(msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT)).isEqualTo("127.0.0.1"); + assertThat(msg.getProperty(MessageConst.PROPERTY_MESSAGE_TTL)).isEqualTo("3000"); + } + + @Test + public void testCreateReplyMessage_Exception() throws MQClientException { + try { + Message msg = MessageUtil.createReplyMessage(createReplyMessage(null), new byte[] {'a'}); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("create reply message fail, requestMessage error, property[" + MessageConst.PROPERTY_CLUSTER + "] is null."); + } + } + + @Test + public void testCreateReplyMessage_reqMsgIsNull() throws MQClientException { + try { + Message msg = MessageUtil.createReplyMessage(null, new byte[] {'a'}); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("create reply message fail, requestMessage cannot be null."); + } + } + + @Test + public void testGetReplyToClient() throws MQClientException { + Message msg = createReplyMessage("clusterName"); + String replyToClient = MessageUtil.getReplyToClient(msg); + assertThat(replyToClient).isNotNull(); + assertThat(replyToClient).isEqualTo("127.0.0.1"); + } + + private Message createReplyMessage(String clusterName) { + Message requestMessage = new Message(); + Map map = new HashMap(); + map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, "127.0.0.1"); + map.put(MessageConst.PROPERTY_CLUSTER, clusterName); + map.put(MessageConst.PROPERTY_MESSAGE_TTL, "3000"); + MessageAccessor.setProperties(requestMessage, map); + return requestMessage; + } + +} diff --git a/common/pom.xml b/common/pom.xml index f6ebf51db4b2bc2fe49b991a165c19d01572c462..129ecb1d6a25f618fd626de9bf42036d2d247fa6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 @@ -41,5 +41,9 @@ org.apache.commons commons-lang3 + + commons-validator + commons-validator + diff --git a/common/src/main/java/org/apache/rocketmq/common/AclConfig.java b/common/src/main/java/org/apache/rocketmq/common/AclConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..191236a0998550f4e62a93d3a2d9b1fdd37eea1f --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/AclConfig.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common; + +import java.util.List; + +public class AclConfig { + + private List globalWhiteAddrs; + + private List plainAccessConfigs; + + + public List getGlobalWhiteAddrs() { + return globalWhiteAddrs; + } + + public void setGlobalWhiteAddrs(List globalWhiteAddrs) { + this.globalWhiteAddrs = globalWhiteAddrs; + } + + public List getPlainAccessConfigs() { + return plainAccessConfigs; + } + + public void setPlainAccessConfigs(List plainAccessConfigs) { + this.plainAccessConfigs = plainAccessConfigs; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 1c3f37d00d61d2fe04630b6a427906544dd4d95d..a7568f0a2079a9f19ee666b362fd21f5a6db3c3f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -61,6 +61,7 @@ public class BrokerConfig { */ private int sendMessageThreadPoolNums = 1; //16 + Runtime.getRuntime().availableProcessors() * 4; private int pullMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2; + private int processReplyMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2; private int queryMessageThreadPoolNums = 8 + Runtime.getRuntime().availableProcessors(); private int adminBrokerThreadPoolNums = 16; @@ -83,6 +84,7 @@ public class BrokerConfig { private boolean fetchNamesrvAddrByAddressServer = false; private int sendThreadPoolQueueCapacity = 10000; private int pullThreadPoolQueueCapacity = 100000; + private int replyThreadPoolQueueCapacity = 10000; private int queryThreadPoolQueueCapacity = 20000; private int clientManagerThreadPoolQueueCapacity = 1000000; private int consumerManagerThreadPoolQueueCapacity = 1000000; @@ -180,6 +182,8 @@ public class BrokerConfig { @ImportantField private boolean aclEnable = false; + private boolean storeReplyMessageEnable = true; + public static String localHostName() { try { return InetAddress.getLocalHost().getHostName(); @@ -374,6 +378,14 @@ public class BrokerConfig { this.pullMessageThreadPoolNums = pullMessageThreadPoolNums; } + public int getProcessReplyMessageThreadPoolNums() { + return processReplyMessageThreadPoolNums; + } + + public void setProcessReplyMessageThreadPoolNums(int processReplyMessageThreadPoolNums) { + this.processReplyMessageThreadPoolNums = processReplyMessageThreadPoolNums; + } + public int getQueryMessageThreadPoolNums() { return queryMessageThreadPoolNums; } @@ -470,6 +482,14 @@ public class BrokerConfig { this.pullThreadPoolQueueCapacity = pullThreadPoolQueueCapacity; } + public int getReplyThreadPoolQueueCapacity() { + return replyThreadPoolQueueCapacity; + } + + public void setReplyThreadPoolQueueCapacity(int replyThreadPoolQueueCapacity) { + this.replyThreadPoolQueueCapacity = replyThreadPoolQueueCapacity; + } + public int getQueryThreadPoolQueueCapacity() { return queryThreadPoolQueueCapacity; } @@ -749,7 +769,7 @@ public class BrokerConfig { public void setMsgTraceTopicName(String msgTraceTopicName) { this.msgTraceTopicName = msgTraceTopicName; } - + public boolean isTraceTopicEnable() { return traceTopicEnable; } @@ -765,4 +785,12 @@ public class BrokerConfig { public void setAclEnable(boolean aclEnable) { this.aclEnable = aclEnable; } + + public boolean isStoreReplyMessageEnable() { + return storeReplyMessageEnable; + } + + public void setStoreReplyMessageEnable(boolean storeReplyMessageEnable) { + this.storeReplyMessageEnable = storeReplyMessageEnable; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index 47cfb6ebd02681b214a0e77fc3e7c30cb1cedb47..43b8f9b8e462e65590bf13e5a82dd31123d38a6a 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.common; public class MQVersion { - public static final int CURRENT_VERSION = Version.V4_5_2.ordinal(); + public static final int CURRENT_VERSION = Version.V4_6_1.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 0af65dff0f11b43e31ae73a8747adf1f72156a0d..de259c931b9c4fbbc0ff97c3fb3cf2ad766ee064 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -45,8 +45,6 @@ import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; public class MixAll { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); - public static final String ROCKETMQ_HOME_ENV = "ROCKETMQ_HOME"; public static final String ROCKETMQ_HOME_PROPERTY = "rocketmq.home.dir"; public static final String NAMESRV_ADDR_ENV = "NAMESRV_ADDR"; @@ -74,27 +72,26 @@ public class MixAll { public static final String CID_ONSAPI_OWNER_GROUP = "CID_ONSAPI_OWNER"; public static final String CID_ONSAPI_PULL_GROUP = "CID_ONSAPI_PULL"; public static final String CID_RMQ_SYS_PREFIX = "CID_RMQ_SYS_"; - public static final List LOCAL_INET_ADDRESS = getLocalInetAddress(); public static final String LOCALHOST = localhost(); public static final String DEFAULT_CHARSET = "UTF-8"; public static final long MASTER_ID = 0L; public static final long CURRENT_JVM_PID = getPID(); - public static final String RETRY_GROUP_TOPIC_PREFIX = "%RETRY%"; - public static final String DLQ_GROUP_TOPIC_PREFIX = "%DLQ%"; + public static final String REPLY_TOPIC_POSTFIX = "REPLY_TOPIC"; public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; public static final String UNIQUE_MSG_QUERY_FLAG = "_UNIQUE_KEY_QUERY"; public static final String DEFAULT_TRACE_REGION_ID = "DefaultRegion"; public static final String CONSUME_CONTEXT_TYPE = "ConsumeContextType"; - public static final String RMQ_SYS_TRANS_HALF_TOPIC = "RMQ_SYS_TRANS_HALF_TOPIC"; public static final String RMQ_SYS_TRACE_TOPIC = "RMQ_SYS_TRACE_TOPIC"; public static final String RMQ_SYS_TRANS_OP_HALF_TOPIC = "RMQ_SYS_TRANS_OP_HALF_TOPIC"; public static final String TRANS_CHECK_MAX_TIME_TOPIC = "TRANS_CHECK_MAX_TIME_TOPIC"; public static final String CID_SYS_RMQ_TRANS = "CID_RMQ_SYS_TRANS"; public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml"; + public static final String REPLY_MESSAGE_FLAG = "reply"; + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); public static String getWSAddr() { String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP); @@ -110,6 +107,10 @@ public class MixAll { return RETRY_GROUP_TOPIC_PREFIX + consumerGroup; } + public static String getReplyTopic(final String clusterName) { + return clusterName + "_" + REPLY_TOPIC_POSTFIX; + } + public static boolean isSysConsumerGroup(final String consumerGroup) { return consumerGroup.startsWith(CID_RMQ_SYS_PREFIX); } @@ -124,8 +125,10 @@ public class MixAll { public static String brokerVIPChannel(final boolean isChange, final String brokerAddr) { if (isChange) { - String[] ipAndPort = brokerAddr.split(":"); - String brokerAddrNew = ipAndPort[0] + ":" + (Integer.parseInt(ipAndPort[1]) - 2); + int split = brokerAddr.lastIndexOf(":"); + String ip = brokerAddr.substring(0, split); + String port = brokerAddr.substring(split + 1); + String brokerAddrNew = ip + ":" + (Integer.parseInt(port) - 2); return brokerAddrNew; } else { return brokerAddr; @@ -416,7 +419,7 @@ public class MixAll { if (address.isLoopbackAddress()) { continue; } - //ip4 highter priority + //ip4 higher priority if (address instanceof Inet6Address) { candidatesHost.add(address.getHostAddress()); continue; diff --git a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java index 68d93f3af3821008591a6385ff16f7ca579df93f..2aab5b79234c8621222b75e43426987c804e30aa 100644 --- a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; import java.net.Inet4Address; +import java.net.Inet6Address; import java.net.InetAddress; import java.net.NetworkInterface; import java.text.NumberFormat; @@ -39,6 +40,7 @@ import java.util.zip.CRC32; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -438,14 +440,21 @@ public class UtilAll { return false; } + public static boolean isInternalV6IP(InetAddress inetAddr) { + if (inetAddr.isAnyLocalAddress() // Wild card ipv6 + || inetAddr.isLinkLocalAddress() // Single broadcast ipv6 address: fe80:xx:xx... + || inetAddr.isLoopbackAddress() //Loopback ipv6 address + || inetAddr.isSiteLocalAddress()) { // Site local ipv6 address: fec0:xx:xx... + return true; + } + return false; + } + private static boolean ipCheck(byte[] ip) { if (ip.length != 4) { throw new RuntimeException("illegal ipv4 bytes"); } -// if (ip[0] == (byte)30 && ip[1] == (byte)10 && ip[2] == (byte)163 && ip[3] == (byte)120) { -// } - if (ip[0] >= (byte) 1 && ip[0] <= (byte) 126) { if (ip[1] == (byte) 1 && ip[2] == (byte) 1 && ip[3] == (byte) 1) { return false; @@ -474,6 +483,15 @@ public class UtilAll { return false; } + private static boolean ipV6Check(byte[] ip) { + if (ip.length != 16) { + throw new RuntimeException("illegal ipv6 bytes"); + } + + InetAddressValidator validator = InetAddressValidator.getInstance(); + return validator.isValidInet6Address(ipToIPv6Str(ip)); + } + public static String ipToIPv4Str(byte[] ip) { if (ip.length != 4) { return null; @@ -483,6 +501,25 @@ public class UtilAll { .append(".").append(ip[3] & 0xFF).toString(); } + public static String ipToIPv6Str(byte[] ip) { + if (ip.length != 16) { + return null; + } + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < ip.length; i++) { + String hex = Integer.toHexString(ip[i] & 0xFF); + if (hex.length() < 2) { + sb.append(0); + } + sb.append(hex); + if (i % 2 == 1 && i < ip.length - 1) { + sb.append(":"); + } + } + return sb.toString(); + } + public static byte[] getIP() { try { Enumeration allNetInterfaces = NetworkInterface.getNetworkInterfaces(); @@ -504,6 +541,15 @@ public class UtilAll { } } } + } else if (ip != null && ip instanceof Inet6Address) { + byte[] ipByte = ip.getAddress(); + if (ipByte.length == 16) { + if (ipV6Check(ipByte)) { + if (!isInternalV6IP(ip)) { + return ipByte; + } + } + } } } } @@ -532,12 +578,12 @@ public class UtilAll { } } - public static String List2String(List list,String splitor) { + public static String list2String(List list, String splitor) { if (list == null || list.size() == 0) { return null; } StringBuffer str = new StringBuffer(); - for (int i = 0;i < list.size();i++) { + for (int i = 0; i < list.size(); i++) { str.append(list.get(i)); if (i == list.size() - 1) { continue; @@ -547,7 +593,7 @@ public class UtilAll { return str.toString(); } - public static List String2List(String str,String splitor) { + public static List string2List(String str, String splitor) { if (StringUtils.isEmpty(str)) { return null; } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageClientIDSetter.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageClientIDSetter.java index d0b202ec64b120a07ecb555a594d8cd9b2d0efc8..df11556383de6c18d18751f43aa5cc7496197697 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageClientIDSetter.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageClientIDSetter.java @@ -31,17 +31,19 @@ public class MessageClientIDSetter { private static long nextStartTime; static { - LEN = 4 + 2 + 4 + 4 + 2; - ByteBuffer tempBuffer = ByteBuffer.allocate(10); - tempBuffer.position(2); - tempBuffer.putInt(UtilAll.getPid()); - tempBuffer.position(0); + byte[] ip; try { - tempBuffer.put(UtilAll.getIP()); + ip = UtilAll.getIP(); } catch (Exception e) { - tempBuffer.put(createFakeIP()); + ip = createFakeIP(); } - tempBuffer.position(6); + LEN = ip.length + 2 + 4 + 4 + 2; + ByteBuffer tempBuffer = ByteBuffer.allocate(ip.length + 2 + 4); + tempBuffer.position(0); + tempBuffer.put(ip); + tempBuffer.position(ip.length); + tempBuffer.putInt(UtilAll.getPid()); + tempBuffer.position(ip.length + 2); tempBuffer.putInt(MessageClientIDSetter.class.getClassLoader().hashCode()); FIX_STRING = UtilAll.bytes2string(tempBuffer.array()); setStartTime(System.currentTimeMillis()); @@ -64,11 +66,12 @@ public class MessageClientIDSetter { public static Date getNearlyTimeFromID(String msgID) { ByteBuffer buf = ByteBuffer.allocate(8); byte[] bytes = UtilAll.string2bytes(msgID); + int ipLength = bytes.length == 28 ? 16 : 4; buf.put((byte) 0); buf.put((byte) 0); buf.put((byte) 0); buf.put((byte) 0); - buf.put(bytes, 10, 4); + buf.put(bytes, ipLength + 2 + 4, 4); buf.position(0); long spanMS = buf.getLong(); Calendar cal = Calendar.getInstance(); @@ -89,13 +92,18 @@ public class MessageClientIDSetter { public static String getIPStrFromID(String msgID) { byte[] ipBytes = getIPFromID(msgID); - return UtilAll.ipToIPv4Str(ipBytes); + if (ipBytes.length == 16) { + return UtilAll.ipToIPv6Str(ipBytes); + } else { + return UtilAll.ipToIPv4Str(ipBytes); + } } public static byte[] getIPFromID(String msgID) { - byte[] result = new byte[4]; byte[] bytes = UtilAll.string2bytes(msgID); - System.arraycopy(bytes, 0, result, 0, 4); + int ipLength = bytes.length == 28 ? 16 : 4; + byte[] result = new byte[ipLength]; + System.arraycopy(bytes, 0, result, 0, ipLength); return result; } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index aa8481643a88252731b195c68f3c6c7337dd1f5a..5bdc846562dc6d294f3a4e5cecc4d4bfd5a83099 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -45,6 +45,13 @@ public class MessageConst { public static final String PROPERTY_TRANSACTION_CHECK_TIMES = "TRANSACTION_CHECK_TIMES"; public static final String PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS = "CHECK_IMMUNITY_TIME_IN_SECONDS"; public static final String PROPERTY_INSTANCE_ID = "INSTANCE_ID"; + public static final String PROPERTY_CORRELATION_ID = "CORRELATION_ID"; + public static final String PROPERTY_MESSAGE_REPLY_TO_CLIENT = "REPLY_TO_CLIENT"; + public static final String PROPERTY_MESSAGE_TTL = "TTL"; + public static final String PROPERTY_REPLY_MESSAGE_ARRIVE_TIME = "ARRIVE_TIME"; + public static final String PROPERTY_PUSH_REPLY_TIME = "PUSH_REPLY_TIME"; + public static final String PROPERTY_CLUSTER = "CLUSTER"; + public static final String PROPERTY_MESSAGE_TYPE = "MSG_TYPE"; public static final String KEY_SEPARATOR = " "; @@ -74,5 +81,12 @@ public class MessageConst { STRING_HASH_SET.add(PROPERTY_MAX_RECONSUME_TIMES); STRING_HASH_SET.add(PROPERTY_CONSUME_START_TIMESTAMP); STRING_HASH_SET.add(PROPERTY_INSTANCE_ID); + STRING_HASH_SET.add(PROPERTY_CORRELATION_ID); + STRING_HASH_SET.add(PROPERTY_MESSAGE_REPLY_TO_CLIENT); + STRING_HASH_SET.add(PROPERTY_MESSAGE_TTL); + STRING_HASH_SET.add(PROPERTY_REPLY_MESSAGE_ARRIVE_TIME); + STRING_HASH_SET.add(PROPERTY_PUSH_REPLY_TIME); + STRING_HASH_SET.add(PROPERTY_CLUSTER); + STRING_HASH_SET.add(PROPERTY_MESSAGE_TYPE); } } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java index af0b638d3d0ff62bed2718708255e52b26d8c85f..54d5e6b60f37cf683bc613176ae62edf0402e9c9 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java @@ -16,9 +16,7 @@ */ package org.apache.rocketmq.common.message; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.sysflag.MessageSysFlag; - +import java.net.Inet4Address; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -29,37 +27,41 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; public class MessageDecoder { - public final static int MSG_ID_LENGTH = 8 + 8; +// public final static int MSG_ID_LENGTH = 8 + 8; public final static Charset CHARSET_UTF8 = Charset.forName("UTF-8"); public final static int MESSAGE_MAGIC_CODE_POSTION = 4; public final static int MESSAGE_FLAG_POSTION = 16; public final static int MESSAGE_PHYSIC_OFFSET_POSTION = 28; - public final static int MESSAGE_STORE_TIMESTAMP_POSTION = 56; + // public final static int MESSAGE_STORE_TIMESTAMP_POSTION = 56; public final static int MESSAGE_MAGIC_CODE = -626843481; public static final char NAME_VALUE_SEPARATOR = 1; public static final char PROPERTY_SEPARATOR = 2; - public static final int PHY_POS_POSITION = 4 + 4 + 4 + 4 + 4 + 8; - public static final int BODY_SIZE_POSITION = 4 // 1 TOTALSIZE - + 4 // 2 MAGICCODE - + 4 // 3 BODYCRC - + 4 // 4 QUEUEID - + 4 // 5 FLAG - + 8 // 6 QUEUEOFFSET - + 8 // 7 PHYSICALOFFSET - + 4 // 8 SYSFLAG - + 8 // 9 BORNTIMESTAMP - + 8 // 10 BORNHOST - + 8 // 11 STORETIMESTAMP - + 8 // 12 STOREHOSTADDRESS - + 4 // 13 RECONSUMETIMES - + 8; // 14 Prepared Transaction Offset + public static final int PHY_POS_POSITION = 4 + 4 + 4 + 4 + 4 + 8; + public static final int SYSFLAG_POSITION = 4 + 4 + 4 + 4 + 4 + 8 + 8; +// public static final int BODY_SIZE_POSITION = 4 // 1 TOTALSIZE +// + 4 // 2 MAGICCODE +// + 4 // 3 BODYCRC +// + 4 // 4 QUEUEID +// + 4 // 5 FLAG +// + 8 // 6 QUEUEOFFSET +// + 8 // 7 PHYSICALOFFSET +// + 4 // 8 SYSFLAG +// + 8 // 9 BORNTIMESTAMP +// + 8 // 10 BORNHOST +// + 8 // 11 STORETIMESTAMP +// + 8 // 12 STOREHOSTADDRESS +// + 4 // 13 RECONSUMETIMES +// + 8; // 14 Prepared Transaction Offset public static String createMessageId(final ByteBuffer input, final ByteBuffer addr, final long offset) { input.flip(); - input.limit(MessageDecoder.MSG_ID_LENGTH); + int msgIDLength = addr.limit() == 8 ? 16 : 28; + input.limit(msgIDLength); input.put(addr); input.putLong(offset); @@ -68,8 +70,9 @@ public class MessageDecoder { } public static String createMessageId(SocketAddress socketAddress, long transactionIdhashCode) { - ByteBuffer byteBuffer = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress; + int msgIDLength = inetSocketAddress.getAddress() instanceof Inet4Address ? 16 : 28; + ByteBuffer byteBuffer = ByteBuffer.allocate(msgIDLength); byteBuffer.put(inetSocketAddress.getAddress().getAddress()); byteBuffer.putInt(inetSocketAddress.getPort()); byteBuffer.putLong(transactionIdhashCode); @@ -80,15 +83,16 @@ public class MessageDecoder { public static MessageId decodeMessageId(final String msgId) throws UnknownHostException { SocketAddress address; long offset; + int ipLength = msgId.length() == 32 ? 4 * 2 : 16 * 2; - byte[] ip = UtilAll.string2bytes(msgId.substring(0, 8)); - byte[] port = UtilAll.string2bytes(msgId.substring(8, 16)); + byte[] ip = UtilAll.string2bytes(msgId.substring(0, ipLength)); + byte[] port = UtilAll.string2bytes(msgId.substring(ipLength, ipLength + 8)); ByteBuffer bb = ByteBuffer.wrap(port); int portInt = bb.getInt(0); address = new InetSocketAddress(InetAddress.getByAddress(ip), portInt); // offset - byte[] data = UtilAll.string2bytes(msgId.substring(16, 32)); + byte[] data = UtilAll.string2bytes(msgId.substring(ipLength + 8, ipLength + 8 + 16)); bb = ByteBuffer.wrap(data); offset = bb.getLong(0); @@ -101,7 +105,24 @@ public class MessageDecoder { * @param byteBuffer msg commit log buffer. */ public static Map decodeProperties(java.nio.ByteBuffer byteBuffer) { - int topicLengthPosition = BODY_SIZE_POSITION + 4 + byteBuffer.getInt(BODY_SIZE_POSITION); + int sysFlag = byteBuffer.getInt(SYSFLAG_POSITION); + int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; + int storehostAddressLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 8 : 20; + int bodySizePosition = 4 // 1 TOTALSIZE + + 4 // 2 MAGICCODE + + 4 // 3 BODYCRC + + 4 // 4 QUEUEID + + 4 // 5 FLAG + + 8 // 6 QUEUEOFFSET + + 8 // 7 PHYSICALOFFSET + + 4 // 8 SYSFLAG + + 8 // 9 BORNTIMESTAMP + + bornhostLength // 10 BORNHOST + + 8 // 11 STORETIMESTAMP + + storehostAddressLength // 12 STOREHOSTADDRESS + + 4 // 13 RECONSUMETIMES + + 8; // 14 Prepared Transaction Offset + int topicLengthPosition = bodySizePosition + 4 + byteBuffer.getInt(bodySizePosition); byte topicLength = byteBuffer.get(topicLengthPosition); @@ -139,6 +160,8 @@ public class MessageDecoder { byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8); short propertiesLength = (short) propertiesBytes.length; int sysFlag = messageExt.getSysFlag(); + int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; + int storehostAddressLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 8 : 20; byte[] newBody = messageExt.getBody(); if (needCompress && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) { newBody = UtilAll.compress(body, 5); @@ -158,9 +181,9 @@ public class MessageDecoder { + 8 // 7 PHYSICALOFFSET + 4 // 8 SYSFLAG + 8 // 9 BORNTIMESTAMP - + 8 // 10 BORNHOST + + bornhostLength // 10 BORNHOST + 8 // 11 STORETIMESTAMP - + 8 // 12 STOREHOSTADDRESS + + storehostAddressLength // 12 STOREHOSTADDRESS + 4 // 13 RECONSUMETIMES + 8 // 14 Prepared Transaction Offset + 4 + bodyLength // 14 BODY @@ -291,8 +314,9 @@ public class MessageDecoder { msgExt.setBornTimestamp(bornTimeStamp); // 10 BORNHOST - byte[] bornHost = new byte[4]; - byteBuffer.get(bornHost, 0, 4); + int bornhostIPLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 : 16; + byte[] bornHost = new byte[bornhostIPLength]; + byteBuffer.get(bornHost, 0, bornhostIPLength); int port = byteBuffer.getInt(); msgExt.setBornHost(new InetSocketAddress(InetAddress.getByAddress(bornHost), port)); @@ -301,8 +325,9 @@ public class MessageDecoder { msgExt.setStoreTimestamp(storeTimestamp); // 12 STOREHOST - byte[] storeHost = new byte[4]; - byteBuffer.get(storeHost, 0, 4); + int storehostIPLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 : 16; + byte[] storeHost = new byte[storehostIPLength]; + byteBuffer.get(storeHost, 0, storehostIPLength); port = byteBuffer.getInt(); msgExt.setStoreHost(new InetSocketAddress(InetAddress.getByAddress(storeHost), port)); @@ -348,7 +373,8 @@ public class MessageDecoder { msgExt.setProperties(map); } - ByteBuffer byteBufferMsgId = ByteBuffer.allocate(MSG_ID_LENGTH); + int msgIDLength = storehostIPLength + 4 + 8; + ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength); String msgId = createMessageId(byteBufferMsgId, msgExt.getStoreHostBytes(), msgExt.getCommitLogOffset()); msgExt.setMsgId(msgId); diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExt.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExt.java index 3f77767eb8d00ff8322df2525b9225034f51f79d..20cb0572717ca138a26ca5d157bbe295dae98fc4 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageExt.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExt.java @@ -16,6 +16,8 @@ */ package org.apache.rocketmq.common.message; +import java.net.Inet4Address; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.ByteBuffer; @@ -66,14 +68,26 @@ public class MessageExt extends Message { public static ByteBuffer socketAddress2ByteBuffer(final SocketAddress socketAddress, final ByteBuffer byteBuffer) { InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress; - byteBuffer.put(inetSocketAddress.getAddress().getAddress(), 0, 4); + InetAddress address = inetSocketAddress.getAddress(); + if (address instanceof Inet4Address) { + byteBuffer.put(inetSocketAddress.getAddress().getAddress(), 0, 4); + } else { + byteBuffer.put(inetSocketAddress.getAddress().getAddress(), 0, 16); + } byteBuffer.putInt(inetSocketAddress.getPort()); byteBuffer.flip(); return byteBuffer; } public static ByteBuffer socketAddress2ByteBuffer(SocketAddress socketAddress) { - ByteBuffer byteBuffer = ByteBuffer.allocate(8); + InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress; + InetAddress address = inetSocketAddress.getAddress(); + ByteBuffer byteBuffer; + if (address instanceof Inet4Address) { + byteBuffer = ByteBuffer.allocate(4 + 4); + } else { + byteBuffer = ByteBuffer.allocate(16 + 4); + } return socketAddress2ByteBuffer(socketAddress, byteBuffer); } @@ -167,6 +181,10 @@ public class MessageExt extends Message { this.sysFlag = sysFlag; } + public void setStoreHostAddressV6Flag() { this.sysFlag = this.sysFlag | MessageSysFlag.STOREHOSTADDRESS_V6_FLAG; } + + public void setBornHostV6Flag() { this.sysFlag = this.sysFlag | MessageSysFlag.BORNHOST_V6_FLAG; } + public int getBodyCRC() { return bodyCRC; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java index 58c4b9fe90f2d2ab4a63961deacf4b2ebffd775a..b3009d738ff80a50de5878ac43815b6943313163 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java @@ -78,6 +78,8 @@ public class RequestCode { public static final int UPDATE_GLOBAL_WHITE_ADDRS_CONFIG = 53; + public static final int GET_BROKER_CLUSTER_ACL_CONFIG = 54; + public static final int PUT_KV_CONFIG = 100; public static final int GET_KV_CONFIG = 101; @@ -180,4 +182,10 @@ public class RequestCode { * resume logic of checking half messages that have been put in TRANS_CHECK_MAXTIME_TOPIC before */ public static final int RESUME_CHECK_HALF_MESSAGE = 323; + + public static final int SEND_REPLY_MESSAGE = 324; + + public static final int SEND_REPLY_MESSAGE_V2 = 325; + + public static final int PUSH_REPLY_MESSAGE_TO_CLIENT = 326; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetBrokerClusterAclConfigResponseBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetBrokerClusterAclConfigResponseBody.java new file mode 100644 index 0000000000000000000000000000000000000000..10ea210c822edee6066a55fef9e3923683836aa3 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetBrokerClusterAclConfigResponseBody.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.common.PlainAccessConfig; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.List; + +public class GetBrokerClusterAclConfigResponseBody extends RemotingSerializable { + + private List globalWhiteAddrs; + + private List plainAccessConfigs; + + public List getGlobalWhiteAddrs() { + return globalWhiteAddrs; + } + + public void setGlobalWhiteAddrs(List globalWhiteAddrs) { + this.globalWhiteAddrs = globalWhiteAddrs; + } + + public List getPlainAccessConfigs() { + return plainAccessConfigs; + } + + public void setPlainAccessConfigs(List plainAccessConfigs) { + this.plainAccessConfigs = plainAccessConfigs; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetBrokerClusterAclConfigResponseHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetBrokerClusterAclConfigResponseHeader.java new file mode 100644 index 0000000000000000000000000000000000000000..dbff54a0e8d3d7795c12e203709d9e58e8ece9bc --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetBrokerClusterAclConfigResponseHeader.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.common.PlainAccessConfig; +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +import java.util.List; + +public class GetBrokerClusterAclConfigResponseHeader implements CommandCustomHeader { + + @CFNotNull + private List plainAccessConfigs; + + @Override + public void checkFields() throws RemotingCommandException { + } + + public List getPlainAccessConfigs() { + return plainAccessConfigs; + } + + public void setPlainAccessConfigs(List plainAccessConfigs) { + this.plainAccessConfigs = plainAccessConfigs; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java new file mode 100644 index 0000000000000000000000000000000000000000..3bb09073f722e85c8be9d5af131e42a4f3592074 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.annotation.CFNullable; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class ReplyMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + @CFNotNull + private String topic; + @CFNotNull + private String defaultTopic; + @CFNotNull + private Integer defaultTopicQueueNums; + @CFNotNull + private Integer queueId; + @CFNotNull + private Integer sysFlag; + @CFNotNull + private Long bornTimestamp; + @CFNotNull + private Integer flag; + @CFNullable + private String properties; + @CFNullable + private Integer reconsumeTimes; + @CFNullable + private boolean unitMode = false; + + @CFNotNull + private String bornHost; + @CFNotNull + private String storeHost; + @CFNotNull + private long storeTimestamp; + + public void checkFields() throws RemotingCommandException { + } + + public String getProducerGroup() { + return producerGroup; + } + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getDefaultTopic() { + return defaultTopic; + } + + public void setDefaultTopic(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + public Integer getDefaultTopicQueueNums() { + return defaultTopicQueueNums; + } + + public void setDefaultTopicQueueNums(Integer defaultTopicQueueNums) { + this.defaultTopicQueueNums = defaultTopicQueueNums; + } + + public Integer getQueueId() { + return queueId; + } + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + public Integer getSysFlag() { + return sysFlag; + } + + public void setSysFlag(Integer sysFlag) { + this.sysFlag = sysFlag; + } + + public Long getBornTimestamp() { + return bornTimestamp; + } + + public void setBornTimestamp(Long bornTimestamp) { + this.bornTimestamp = bornTimestamp; + } + + public Integer getFlag() { + return flag; + } + + public void setFlag(Integer flag) { + this.flag = flag; + } + + public String getProperties() { + return properties; + } + + public void setProperties(String properties) { + this.properties = properties; + } + + public Integer getReconsumeTimes() { + return reconsumeTimes; + } + + public void setReconsumeTimes(Integer reconsumeTimes) { + this.reconsumeTimes = reconsumeTimes; + } + + public boolean isUnitMode() { + return unitMode; + } + + public void setUnitMode(boolean unitMode) { + this.unitMode = unitMode; + } + + public String getBornHost() { + return bornHost; + } + + public void setBornHost(String bornHost) { + this.bornHost = bornHost; + } + + public String getStoreHost() { + return storeHost; + } + + public void setStoreHost(String storeHost) { + this.storeHost = storeHost; + } + + public long getStoreTimestamp() { + return storeTimestamp; + } + + public void setStoreTimestamp(long storeTimestamp) { + this.storeTimestamp = storeTimestamp; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java index bd575baba8caa52fa3df174a3cd04c9c6ccade74..6304ea2c4a37f5696f08e85614d003e363d4347a 100644 --- a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java +++ b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java @@ -152,6 +152,9 @@ public class StatsItem { public void samplingInSeconds() { synchronized (this.csListMinute) { + if (this.csListMinute.size() == 0) { + this.csListMinute.add(new CallSnapshot(System.currentTimeMillis() - 10 * 1000, 0, 0)); + } this.csListMinute.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value .get())); if (this.csListMinute.size() > 7) { @@ -162,6 +165,9 @@ public class StatsItem { public void samplingInMinutes() { synchronized (this.csListHour) { + if (this.csListHour.size() == 0) { + this.csListHour.add(new CallSnapshot(System.currentTimeMillis() - 10 * 60 * 1000, 0, 0)); + } this.csListHour.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value .get())); if (this.csListHour.size() > 7) { @@ -172,6 +178,9 @@ public class StatsItem { public void samplingInHour() { synchronized (this.csListDay) { + if (this.csListDay.size() == 0) { + this.csListDay.add(new CallSnapshot(System.currentTimeMillis() - 1 * 60 * 60 * 1000, 0, 0)); + } this.csListDay.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value .get())); if (this.csListDay.size() > 25) { diff --git a/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java b/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java index f1397706cc5e924ae4a8019045fd83a98643c20c..d534571eb55276f2452780a13bb6148ea8f5dd43 100644 --- a/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java +++ b/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java @@ -23,6 +23,8 @@ public class MessageSysFlag { public final static int TRANSACTION_PREPARED_TYPE = 0x1 << 2; public final static int TRANSACTION_COMMIT_TYPE = 0x2 << 2; public final static int TRANSACTION_ROLLBACK_TYPE = 0x3 << 2; + public final static int BORNHOST_V6_FLAG = 0x1 << 4; + public final static int STOREHOSTADDRESS_V6_FLAG = 0x1 << 5; public static int getTransactionValue(final int flag) { return flag & TRANSACTION_ROLLBACK_TYPE; @@ -35,4 +37,5 @@ public class MessageSysFlag { public static int clearCompressedFlag(final int flag) { return flag & (~COMPRESSED_FLAG); } + } diff --git a/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java b/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java index d476a35b749348b4b279003e252b1e83358c09ad..ce7558f2bcffd66b02b2b15b8a1e0a3306bce459 100644 --- a/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java +++ b/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java @@ -21,6 +21,7 @@ public class PullSysFlag { private final static int FLAG_SUSPEND = 0x1 << 1; private final static int FLAG_SUBSCRIPTION = 0x1 << 2; private final static int FLAG_CLASS_FILTER = 0x1 << 3; + private final static int FLAG_LITE_PULL_MESSAGE = 0x1 << 4; public static int buildSysFlag(final boolean commitOffset, final boolean suspend, final boolean subscription, final boolean classFilter) { @@ -45,6 +46,17 @@ public class PullSysFlag { return flag; } + public static int buildSysFlag(final boolean commitOffset, final boolean suspend, + final boolean subscription, final boolean classFilter, final boolean litePull) { + int flag = buildSysFlag(commitOffset, suspend, subscription, classFilter); + + if (litePull) { + flag |= FLAG_LITE_PULL_MESSAGE; + } + + return flag; + } + public static int clearCommitOffsetFlag(final int sysFlag) { return sysFlag & (~FLAG_COMMIT_OFFSET); } @@ -64,4 +76,8 @@ public class PullSysFlag { public static boolean hasClassFilterFlag(final int sysFlag) { return (sysFlag & FLAG_CLASS_FILTER) == FLAG_CLASS_FILTER; } + + public static boolean hasLitePullFlag(final int sysFlag) { + return (sysFlag & FLAG_LITE_PULL_MESSAGE) == FLAG_LITE_PULL_MESSAGE; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/CorrelationIdUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/CorrelationIdUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..86d1fd4d42dbf21db700df4ce84667c2d4980287 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/utils/CorrelationIdUtil.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.utils; + +import java.util.UUID; + +public class CorrelationIdUtil { + public static String createCorrelationId() { + return UUID.randomUUID().toString(); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java index 265645213e4fe6f3ba4a6aefc73720013129d0e3..b854099a5bd757a984d402e86d6da06b0119ae12 100644 --- a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java @@ -17,6 +17,8 @@ package org.apache.rocketmq.common; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Properties; import org.junit.Test; @@ -98,6 +100,15 @@ public class UtilAllTest { assertThat(UtilAll.isBlank("Hello")).isFalse(); } + @Test + public void testIPv6Check() throws UnknownHostException { + InetAddress nonInternal = InetAddress.getByName("2408:4004:0180:8100:3FAA:1DDE:2B3F:898A"); + InetAddress internal = InetAddress.getByName("FE80:0000:0000:0000:0000:0000:0000:FFFF"); + assertThat(UtilAll.isInternalV6IP(nonInternal)).isFalse(); + assertThat(UtilAll.isInternalV6IP(internal)).isTrue(); + assertThat(UtilAll.ipToIPv6Str(nonInternal.getAddress()).toUpperCase()).isEqualTo("2408:4004:0180:8100:3FAA:1DDE:2B3F:898A"); + } + static class DemoConfig { private int demoWidth = 0; private int demoLength = 0; diff --git a/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java b/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java new file mode 100644 index 0000000000000000000000000000000000000000..1ec6d93cffc2391a9e049b545f5b3f3956585cdb --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.message; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class MessageClientIDSetterTest { + + @Test + public void testGetIPStrFromID() { + String ipv4HostMsgId = "C0A803CA00002A9F0000000000031367"; + String ipv6HostMsgId = "24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"; + String v4Ip = "192.168.3.202"; + String v6Ip = "2408:4004:0180:8100:3faa:1dde:2b3f:898a"; + assertThat(MessageClientIDSetter.getIPStrFromID(ipv4HostMsgId)).isEqualTo(v4Ip); + assertThat(MessageClientIDSetter.getIPStrFromID(ipv6HostMsgId)).isEqualTo(v6Ip); + } + +} diff --git a/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java b/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java index d14d6b060c89168cde510531da86740ec8b33ad5..82ad2714dd2c255eae0067a587aa3467554b0b7f 100644 --- a/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java @@ -25,6 +25,7 @@ import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.Map; +import static org.apache.rocketmq.common.message.MessageDecoder.createMessageId; import static org.assertj.core.api.Assertions.assertThat; public class MessageDecoderTest { @@ -77,4 +78,172 @@ public class MessageDecoderTest { assertThat("hello").isEqualTo(properties.get("b")); assertThat("3.14").isEqualTo(properties.get("c")); } + + @Test + public void testDecodePropertiesOnIPv6Host() { + MessageExt messageExt = new MessageExt(); + + messageExt.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + messageExt.setBornHostV6Flag(); + messageExt.setStoreHostAddressV6Flag(); + messageExt.setTopic("abc"); + messageExt.setBody("hello!q!".getBytes()); + try { + messageExt.setBornHost(new InetSocketAddress(InetAddress.getByName("1050:0000:0000:0000:0005:0600:300c:326b"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + messageExt.setBornTimestamp(System.currentTimeMillis()); + messageExt.setCommitLogOffset(123456); + messageExt.setPreparedTransactionOffset(0); + messageExt.setQueueId(0); + messageExt.setQueueOffset(123); + messageExt.setReconsumeTimes(0); + try { + messageExt.setStoreHost(new InetSocketAddress(InetAddress.getByName("::1"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + + messageExt.putUserProperty("a", "123"); + messageExt.putUserProperty("b", "hello"); + messageExt.putUserProperty("c", "3.14"); + + byte[] msgBytes = new byte[0]; + try { + msgBytes = MessageDecoder.encode(messageExt, false); + } catch (Exception e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + + ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length); + byteBuffer.put(msgBytes); + + Map properties = MessageDecoder.decodeProperties(byteBuffer); + + assertThat(properties).isNotNull(); + assertThat("123").isEqualTo(properties.get("a")); + assertThat("hello").isEqualTo(properties.get("b")); + assertThat("3.14").isEqualTo(properties.get("c")); + } + + @Test + public void testEncodeAndDecode() { + MessageExt messageExt = new MessageExt(); + + messageExt.setMsgId("645100FA00002A9F000000489A3AA09E"); + messageExt.setTopic("abc"); + messageExt.setBody("hello!q!".getBytes()); + try { + messageExt.setBornHost(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + messageExt.setBornTimestamp(System.currentTimeMillis()); + messageExt.setCommitLogOffset(123456); + messageExt.setPreparedTransactionOffset(0); + messageExt.setQueueId(1); + messageExt.setQueueOffset(123); + messageExt.setReconsumeTimes(0); + try { + messageExt.setStoreHost(new InetSocketAddress(InetAddress.getLocalHost(), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + + messageExt.putUserProperty("a", "123"); + messageExt.putUserProperty("b", "hello"); + messageExt.putUserProperty("c", "3.14"); + + byte[] msgBytes = new byte[0]; + try { + msgBytes = MessageDecoder.encode(messageExt, false); + } catch (Exception e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + + ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length); + byteBuffer.put(msgBytes); + + byteBuffer.clear(); + MessageExt decodedMsg = MessageDecoder.decode(byteBuffer); + + assertThat(decodedMsg).isNotNull(); + assertThat(1).isEqualTo(decodedMsg.getQueueId()); + assertThat(123456L).isEqualTo(decodedMsg.getCommitLogOffset()); + assertThat("hello!q!".getBytes()).isEqualTo(decodedMsg.getBody()); + + int msgIDLength = 4 + 4 + 8; + ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength); + String msgId = createMessageId(byteBufferMsgId, messageExt.getStoreHostBytes(), messageExt.getCommitLogOffset()); + assertThat(msgId).isEqualTo(decodedMsg.getMsgId()); + + assertThat("abc").isEqualTo(decodedMsg.getTopic()); + } + + @Test + public void testEncodeAndDecodeOnIPv6Host() { + MessageExt messageExt = new MessageExt(); + + messageExt.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + messageExt.setBornHostV6Flag(); + messageExt.setStoreHostAddressV6Flag(); + messageExt.setTopic("abc"); + messageExt.setBody("hello!q!".getBytes()); + try { + messageExt.setBornHost(new InetSocketAddress(InetAddress.getByName("1050:0000:0000:0000:0005:0600:300c:326b"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + messageExt.setBornTimestamp(System.currentTimeMillis()); + messageExt.setCommitLogOffset(123456); + messageExt.setPreparedTransactionOffset(0); + messageExt.setQueueId(1); + messageExt.setQueueOffset(123); + messageExt.setReconsumeTimes(0); + try { + messageExt.setStoreHost(new InetSocketAddress(InetAddress.getByName("::1"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + + messageExt.putUserProperty("a", "123"); + messageExt.putUserProperty("b", "hello"); + messageExt.putUserProperty("c", "3.14"); + + byte[] msgBytes = new byte[0]; + try { + msgBytes = MessageDecoder.encode(messageExt, false); + } catch (Exception e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + + ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length); + byteBuffer.put(msgBytes); + + byteBuffer.clear(); + MessageExt decodedMsg = MessageDecoder.decode(byteBuffer); + + assertThat(decodedMsg).isNotNull(); + assertThat(1).isEqualTo(decodedMsg.getQueueId()); + assertThat(123456L).isEqualTo(decodedMsg.getCommitLogOffset()); + assertThat("hello!q!".getBytes()).isEqualTo(decodedMsg.getBody()); + assertThat(48).isEqualTo(decodedMsg.getSysFlag()); + + int msgIDLength = 16 + 4 + 8; + ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength); + String msgId = createMessageId(byteBufferMsgId, messageExt.getStoreHostBytes(), messageExt.getCommitLogOffset()); + assertThat(msgId).isEqualTo(decodedMsg.getMsgId()); + + assertThat("abc").isEqualTo(decodedMsg.getTopic()); + } } diff --git a/common/src/test/java/org/apache/rocketmq/common/stats/StatsItemSetTest.java b/common/src/test/java/org/apache/rocketmq/common/stats/StatsItemSetTest.java index bd6550d7be7fced55f8d925ae4cb6efe84562d5d..4b4a86765b99aa4bba2437c3e1f2eab7de27facf 100644 --- a/common/src/test/java/org/apache/rocketmq/common/stats/StatsItemSetTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/stats/StatsItemSetTest.java @@ -44,6 +44,35 @@ public class StatsItemSetTest { assertEquals(10, test_unit_moment().longValue()); } + @Test + public void test_statsOfFirstStatisticsCycle() throws InterruptedException { + final StatsItemSet statsItemSet = new StatsItemSet("topicTest", scheduler, null); + executor = new ThreadPoolExecutor(10, 20, 10, TimeUnit.SECONDS, + new ArrayBlockingQueue(100), new ThreadFactoryImpl("testMultiThread")); + for (int i = 0; i < 10; i++) { + executor.submit(new Runnable() { + @Override + public void run() { + statsItemSet.addValue("topicTest", 2, 1); + } + }); + } + while (true) { + if (executor.getCompletedTaskCount() == 10) { + break; + } + Thread.sleep(1000); + } + // simulate schedule task execution + statsItemSet.getStatsItem("topicTest").samplingInSeconds(); + statsItemSet.getStatsItem("topicTest").samplingInMinutes(); + statsItemSet.getStatsItem("topicTest").samplingInHour(); + + assertEquals(20L, statsItemSet.getStatsDataInMinute("topicTest").getSum()); + assertEquals(20L, statsItemSet.getStatsDataInHour("topicTest").getSum()); + assertEquals(20L, statsItemSet.getStatsDataInDay("topicTest").getSum()); + } + private AtomicLong test_unit() throws InterruptedException { final StatsItemSet statsItemSet = new StatsItemSet("topicTest", scheduler, null); executor = new ThreadPoolExecutor(10, 20, 10, TimeUnit.SECONDS, diff --git a/common/src/test/java/org/apache/rocketmq/common/sysflag/PullSysFlagTest.java b/common/src/test/java/org/apache/rocketmq/common/sysflag/PullSysFlagTest.java new file mode 100644 index 0000000000000000000000000000000000000000..60e18123c47ceae127e44c77d697cb7f822e6c2f --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/sysflag/PullSysFlagTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.sysflag; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PullSysFlagTest { + + @Test + public void testLitePullFlag() { + int flag = PullSysFlag.buildSysFlag(false, false, false, false, true); + assertThat(PullSysFlag.hasLitePullFlag(flag)).isTrue(); + } + + @Test + public void testLitePullFlagFalse() { + int flag = PullSysFlag.buildSysFlag(false, false, false, false, false); + assertThat(PullSysFlag.hasLitePullFlag(flag)).isFalse(); + } +} diff --git a/distribution/pom.xml b/distribution/pom.xml index 290506c7f7bcb94c48171d6313da54206c91b340..73dc8e421191bbdd5292dd70c98eceade87758e3 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/docs/cn/README b/docs/cn/README new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/docs/cn/README.md b/docs/cn/README.md index c8bdc8dc0d3f1ac460b551e213caf8cdb77a948d..2dbd8549133d97588b48fe4c0746d578291af123 100644 --- a/docs/cn/README.md +++ b/docs/cn/README.md @@ -1,7 +1,7 @@ Apache RocketMQ开发者指南 -------- -##### 这个开发者指南是帮忙您快速了解,并使用 Apache RocketMQ +##### 这个开发者指南是帮助您快速了解,并使用 Apache RocketMQ ### 1. 概念和特性 @@ -19,7 +19,7 @@ ### 3. 样例 -- [样例(Example)](RocketMQ_Example.md) ：介绍RocketMQ的常见用法，包括基本样例、顺序消息样例、延时消息样例、批量消息样例、过滤消息样例、事物消息样例等。 +- [样例(Example)](RocketMQ_Example.md) ：介绍RocketMQ的常见用法，包括基本样例、顺序消息样例、延时消息样例、批量消息样例、过滤消息样例、事务消息样例等。 ### 4. 最佳实践 diff --git a/docs/cn/RocketMQ_Example.md b/docs/cn/RocketMQ_Example.md index d298db84203d99eef96a9e50bc45d46b179dff8a..09acb0de9740e15a2c41ab492e70dcfa18e1acaa 100644 --- a/docs/cn/RocketMQ_Example.md +++ b/docs/cn/RocketMQ_Example.md @@ -66,7 +66,11 @@ public class AsyncProducer { // 启动Producer实例 producer.start(); producer.setRetryTimesWhenSendAsyncFailed(0); - for (int i = 0; i < 100; i++) { + + int messageCount = 100; + // 根据消息数量实例化倒计时计算器 + final CountDownLatch2 countDownLatch = new CountDownLatch2(messageCount); + for (int i = 0; i < messageCount; i++) { final int index = i; // 创建消息，并指定Topic，Tag和消息体 Message msg = new Message("TopicTest", @@ -87,6 +91,8 @@ public class AsyncProducer { } }); } + // 等待5s + countDownLatch.await(5, TimeUnit.SECONDS); // 如果不再发送消息，关闭Producer实例。 producer.shutdown(); } diff --git a/docs/cn/acl/user_guide.md b/docs/cn/acl/user_guide.md index 01c37dc85bd15f9d424abb39aba31167aa7a06a0..4ee03ba60bd8972ed92c4774f207d55291936323 100644 --- a/docs/cn/acl/user_guide.md +++ b/docs/cn/acl/user_guide.md @@ -152,6 +152,18 @@ sh mqadmin clusterAclConfigVersion -n 192.168.1.2:9876 -c DefaultCluster | c | eg:DefaultCluster | 指定集群名称(与broker地址二选一) | | b | eg:192.168.12.134:10911 | 指定broker地址(与集群名称二选一) | +### 7.5 查询集群/Broker的ACL配置文件全部内容 +该命令的示例如下： + +sh mqadmin getAccessConfigSubCommand -n 192.168.1.2:9876 -c DefaultCluster + +说明：如果指定的是集群名称，则会在集群中各个broker节点执行该命令；否则会在单个broker节点执行该命令。 + +| 参数 | 取值 | 含义 | +| --- | --- | --- | +| n | eg:192.168.1.2:9876 | namesrv地址(必填) | +| c | eg:DefaultCluster | 指定集群名称(与broker地址二选一) | +| b | eg:192.168.12.134:10911 | 指定broker地址(与集群名称二选一) | **特别注意**开启Acl鉴权认证后导致Master/Slave和Dledger模式下Broker同步数据异常的问题， 在社区[4.5.1]版本中已经修复，具体的PR链接为：https://github.com/apache/rocketmq/pull/1149； \ No newline at end of file diff --git a/docs/cn/concept.md b/docs/cn/concept.md index b486b38195d68a48a32ded7afe061c1db0216eea..8ad0be6da808a843b8c5fcdf3a03b00c1bc8032f 100644 --- a/docs/cn/concept.md +++ b/docs/cn/concept.md @@ -26,7 +26,7 @@ RocketMQ主要由 Producer、Broker、Consumer 三部分组成，其中Producer Consumer消费的一种类型，该模式下Broker收到数据后会主动推送给消费端，该消费模式一般实时性较高。 ## 9 生产者组（Producer Group） - 同一类Producer的集合，这类Producer发送同一类消息且发送逻辑一致。如果发送的是事物消息且原始生产者在发送之后崩溃，则Broker服务器会联系同一生产者组的其他生产者实例以提交或回溯消费。 + 同一类Producer的集合，这类Producer发送同一类消息且发送逻辑一致。如果发送的是事务消息且原始生产者在发送之后崩溃，则Broker服务器会联系同一生产者组的其他生产者实例以提交或回溯消费。 ## 10 消费者组（Consumer Group） 同一类Consumer的集合，这类Consumer通常消费同一类消息且消费逻辑一致。消费者组使得在消息消费方面，实现负载均衡和容错的目标变得非常容易。要注意的是，消费者组的消费者实例必须订阅完全相同的Topic。RocketMQ 支持两种消息模式：集群消费（Clustering）和广播消费（Broadcasting）。 diff --git a/docs/cn/design.md b/docs/cn/design.md index e30b466f0613d5fdf60b9d91303cb9723bd81728..dcd39026f2648e85edd3ad2ae54e7d3fa6b76f72 100644 --- a/docs/cn/design.md +++ b/docs/cn/design.md @@ -32,7 +32,7 @@ (2) 异步刷盘：能够充分利用OS的PageCache的优势，只要消息写入PageCache即可将成功的ACK返回给Producer端。消息刷盘采用后台异步线程提交的方式进行，降低了读写延迟，提高了MQ的性能和吞吐量。 ### 2 通信机制 -RocketMQ消息队列集群主要包括NameServe、Broker(Master/Slave)、Producer、Consumer4个角色，基本通讯流程如下： +RocketMQ消息队列集群主要包括NameServer、Broker(Master/Slave)、Producer、Consumer4个角色，基本通讯流程如下： (1) Broker启动后需要完成一次将自己注册至NameServer的操作；随后每隔30s时间定时向NameServer上报Topic路由信息。 @@ -57,7 +57,7 @@ Header字段 | 类型 | Request说明 | Response说明 code |int | 请求操作码，应答方根据不同的请求码进行不同的业务处理 | 应答响应码。0表示成功，非0则表示各种错误 language | LanguageCode | 请求方实现的语言 | 应答方实现的语言 version | int | 请求方程序的版本 | 应答方程序的版本 -opaque | int |相当于reqeustId，在同一个连接上的不同请求标识码，与响应消息中的相对应 | 应答不做修改直接返回 +opaque | int |相当于requestId，在同一个连接上的不同请求标识码，与响应消息中的相对应 | 应答不做修改直接返回 flag | int | 区分是普通RPC还是onewayRPC得标志 | 区分是普通RPC还是onewayRPC得标志 remark | String | 传输自定义文本信息 | 传输自定义文本信息 extFields | HashMap | 请求自定义扩展信息 | 响应自定义扩展信息 @@ -95,7 +95,7 @@ M1 | NettyServerCodecThread_%d | Worker线程池 M2 | RemotingExecutorThread_%d | 业务processor处理线程池 ### 3 消息过滤 -RocketMQ分布式消息队列的消息过滤方式有别于其它MQ中间件，是在Consumer端订阅消息时再做消息过滤的。RocketMQ这么做是还是在于其Producer端写入消息和Consomer端订阅消息采用分离存储的机制来实现的，Consumer端订阅消息是需要通过ConsumeQueue这个消息消费的逻辑队列拿到一个索引，然后再从CommitLog里面读取真正的消息实体内容，所以说到底也是还绕不开其存储结构。其ConsumeQueue的存储结构如下，可以看到其中有8个字节存储的Message Tag的哈希值，基于Tag的消息过滤正式基于这个字段值的。 +RocketMQ分布式消息队列的消息过滤方式有别于其它MQ中间件，是在Consumer端订阅消息时再做消息过滤的。RocketMQ这么做是在于其Producer端写入消息和Consumer端订阅消息采用分离存储的机制来实现的，Consumer端订阅消息是需要通过ConsumeQueue这个消息消费的逻辑队列拿到一个索引，然后再从CommitLog里面读取真正的消息实体内容，所以说到底也是还绕不开其存储结构。其ConsumeQueue的存储结构如下，可以看到其中有8个字节存储的Message Tag的哈希值，基于Tag的消息过滤正式基于这个字段值的。 ![](image/rocketmq_design_7.png) diff --git a/docs/cn/features.md b/docs/cn/features.md index 859e0f84a3464f1377d27304d0f544da2e9deb17..e7b63864e9c6e2032bd95d9eb078d62d1d8ca51a 100644 --- a/docs/cn/features.md +++ b/docs/cn/features.md @@ -16,7 +16,7 @@ RocketMQ的消费者可以根据Tag进行消息过滤，也支持自定义属性过滤。消息过滤目前是在Broker端实现的，优点是减少了对于Consumer无用消息的网络传输，缺点是增加了Broker的负担、而且实现相对复杂。 ## 4 消息可靠性 RocketMQ支持消息的高可靠，影响消息可靠性的几种情况： -1) Broker正常关闭 +1) Broker非正常关闭 2) Broker异常Crash 3) OS Crash 4) 机器掉电，但是能立即恢复供电情况 diff --git a/docs/cn/operation.md b/docs/cn/operation.md index ae85a30b2320d5e0387ab31202d657b28cf966bc..e35bd6812dbf117c268382873ae2d9cbfe3e11be 100644 --- a/docs/cn/operation.md +++ b/docs/cn/operation.md @@ -1033,30 +1033,6 @@ $ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker -s 是否执行jstack - - getConsumerStatus - 获取 Consumer 消费进度 - -g - 消费者所属组名 - - - -t - 查询主题 - - - -i - Consumer 客户端 ip - - - -n - NameServer 服务地址，格式 ip:port - - - -h - 打印帮助 - updateSubGroup diff --git a/docs/en/Configuration_Client.md b/docs/en/Configuration_Client.md index 4b6d2fe39a536c0057c925e634961b66b613777e..dedb4240b7cf2ca614d665d351c3c7e7ea9aa42c 100644 --- a/docs/en/Configuration_Client.md +++ b/docs/en/Configuration_Client.md @@ -25,7 +25,7 @@ export NAMESRV_ADDR=192.168.0.1:9876;192.168.0.2:9876 ``` - HTTP static server addressing(default) -After client started, it will access a http static server address, as: , this URL return the following contents: +After client started, it will access the http static server address, as: , this URL return the following contents: ```text 192.168.0.1:9876;192.168.0.2:9876 diff --git a/example/pom.xml b/example/pom.xml index f55bb718ddd4cb5f6a9dac51ea1460da23eb2237..e93eb4bbf9fea345bd5c912c44518752a872b573 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 @@ -51,12 +51,12 @@ org.apache.rocketmq rocketmq-openmessaging - 4.5.2 + 4.6.1 org.apache.rocketmq rocketmq-acl - 4.5.2 + 4.6.1 diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java index d431d3ecc6ad6c05dae4ebe8b7796d930c4905fb..08897faa2636359e8bb5c982afd71c7a265fbb4a 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java @@ -22,7 +22,9 @@ import java.util.LinkedList; import java.util.List; import java.util.Timer; import java.util.TimerTask; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicLong; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; @@ -49,15 +51,16 @@ public class Consumer { final String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : "BenchmarkTest"; final String groupPrefix = commandLine.hasOption('g') ? commandLine.getOptionValue('g').trim() : "benchmark_consumer"; - final String isPrefixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : "true"; + final String isSuffixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : "true"; final String filterType = commandLine.hasOption('f') ? commandLine.getOptionValue('f').trim() : null; final String expression = commandLine.hasOption('e') ? commandLine.getOptionValue('e').trim() : null; + final double failRate = commandLine.hasOption('r') ? Double.parseDouble(commandLine.getOptionValue('r').trim()) : 0.0; String group = groupPrefix; - if (Boolean.parseBoolean(isPrefixEnable)) { - group = groupPrefix + "_" + Long.toString(System.currentTimeMillis() % 100); + if (Boolean.parseBoolean(isSuffixEnable)) { + group = groupPrefix + "_" + (System.currentTimeMillis() % 100); } - System.out.printf("topic: %s, group: %s, prefix: %s, filterType: %s, expression: %s%n", topic, group, isPrefixEnable, filterType, expression); + System.out.printf("topic: %s, group: %s, suffix: %s, filterType: %s, expression: %s%n", topic, group, isSuffixEnable, filterType, expression); final StatsBenchmarkConsumer statsBenchmarkConsumer = new StatsBenchmarkConsumer(); @@ -85,9 +88,15 @@ public class Consumer { (long) (((end[1] - begin[1]) / (double) (end[0] - begin[0])) * 1000L); final double averageB2CRT = (end[2] - begin[2]) / (double) (end[1] - begin[1]); final double averageS2CRT = (end[3] - begin[3]) / (double) (end[1] - begin[1]); + final long failCount = end[4] - begin[4]; + final long b2cMax = statsBenchmarkConsumer.getBorn2ConsumerMaxRT().get(); + final long s2cMax = statsBenchmarkConsumer.getStore2ConsumerMaxRT().get(); + + statsBenchmarkConsumer.getBorn2ConsumerMaxRT().set(0); + statsBenchmarkConsumer.getStore2ConsumerMaxRT().set(0); - System.out.printf("Consume TPS: %d Average(B2C) RT: %7.3f Average(S2C) RT: %7.3f MAX(B2C) RT: %d MAX(S2C) RT: %d%n", - consumeTps, averageB2CRT, averageS2CRT, end[4], end[5] + System.out.printf("TPS: %d FAIL: %d AVG(B2C) RT: %7.3f AVG(S2C) RT: %7.3f MAX(B2C) RT: %d MAX(S2C) RT: %d%n", + consumeTps, failCount, averageB2CRT, averageS2CRT, b2cMax, s2cMax ); } } @@ -103,6 +112,10 @@ public class Consumer { }, 10000, 10000); DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group); + if (commandLine.hasOption('n')) { + String ns = commandLine.getOptionValue('n'); + consumer.setNamesrvAddr(ns); + } consumer.setInstanceName(Long.toString(System.currentTimeMillis())); if (filterType == null || expression == null) { @@ -140,7 +153,12 @@ public class Consumer { compareAndSetMax(statsBenchmarkConsumer.getStore2ConsumerMaxRT(), store2ConsumerRT); - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + if (ThreadLocalRandom.current().nextDouble() < failRate) { + statsBenchmarkConsumer.getFailCount().incrementAndGet(); + return ConsumeConcurrentlyStatus.RECONSUME_LATER; + } else { + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } } }); @@ -170,6 +188,10 @@ public class Consumer { opt.setRequired(false); options.addOption(opt); + opt = new Option("r", "fail rate", true, "consumer fail rate, default 0"); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -196,14 +218,15 @@ class StatsBenchmarkConsumer { private final AtomicLong store2ConsumerMaxRT = new AtomicLong(0L); + private final AtomicLong failCount = new AtomicLong(0L); + public Long[] createSnapshot() { Long[] snap = new Long[] { System.currentTimeMillis(), this.receiveMessageTotalCount.get(), this.born2ConsumerTotalRT.get(), this.store2ConsumerTotalRT.get(), - this.born2ConsumerMaxRT.get(), - this.store2ConsumerMaxRT.get(), + this.failCount.get() }; return snap; @@ -228,4 +251,8 @@ class StatsBenchmarkConsumer { public AtomicLong getStore2ConsumerMaxRT() { return store2ConsumerMaxRT; } + + public AtomicLong getFailCount() { + return failCount; + } } diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java index d9fafdd08e88ab4fcdad9226698f573db2ca7553..3531eb528fdf55de4d82d43fca9b36345903e15a 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java @@ -18,43 +18,64 @@ package org.apache.rocketmq.example.benchmark; import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedHashMap; import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.LocalTransactionExecuter; import org.apache.rocketmq.client.producer.LocalTransactionState; import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.producer.TransactionCheckListener; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.client.producer.TransactionMQProducer; import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.srvutil.ServerUtil; public class TransactionProducer { - private static int threadCount; - private static int messageSize; - private static boolean ischeck; - private static boolean ischeckffalse; - - public static void main(String[] args) throws MQClientException, UnsupportedEncodingException { - threadCount = args.length >= 1 ? Integer.parseInt(args[0]) : 32; - messageSize = args.length >= 2 ? Integer.parseInt(args[1]) : 1024 * 2; - ischeck = args.length >= 3 && Boolean.parseBoolean(args[2]); - ischeckffalse = args.length >= 4 && Boolean.parseBoolean(args[3]); + private static final long START_TIME = System.currentTimeMillis(); + private static final AtomicLong MSG_COUNT = new AtomicLong(0); - final Message msg = buildMessage(messageSize); + //broker max check times should less than this value + static final int MAX_CHECK_RESULT_IN_MSG = 20; - final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount); + public static void main(String[] args) throws MQClientException, UnsupportedEncodingException { + Options options = ServerUtil.buildCommandlineOptions(new Options()); + CommandLine commandLine = ServerUtil.parseCmdLine("TransactionProducer", args, buildCommandlineOptions(options), new PosixParser()); + TxSendConfig config = new TxSendConfig(); + config.topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : "BenchmarkTest"; + config.threadCount = commandLine.hasOption('w') ? Integer.parseInt(commandLine.getOptionValue('w')) : 32; + config.messageSize = commandLine.hasOption('s') ? Integer.parseInt(commandLine.getOptionValue('s')) : 2048; + config.sendRollbackRate = commandLine.hasOption("sr") ? Double.parseDouble(commandLine.getOptionValue("sr")) : 0.0; + config.sendUnknownRate = commandLine.hasOption("su") ? Double.parseDouble(commandLine.getOptionValue("su")) : 0.0; + config.checkRollbackRate = commandLine.hasOption("cr") ? Double.parseDouble(commandLine.getOptionValue("cr")) : 0.0; + config.checkUnknownRate = commandLine.hasOption("cu") ? Double.parseDouble(commandLine.getOptionValue("cu")) : 0.0; + config.batchId = commandLine.hasOption("b") ? Long.parseLong(commandLine.getOptionValue("b")) : System.currentTimeMillis(); + config.sendInterval = commandLine.hasOption("i") ? Integer.parseInt(commandLine.getOptionValue("i")) : 0; + + final ExecutorService sendThreadPool = Executors.newFixedThreadPool(config.threadCount); final StatsBenchmarkTProducer statsBenchmark = new StatsBenchmarkTProducer(); final Timer timer = new Timer("BenchmarkTimerThread", true); - final LinkedList snapshotList = new LinkedList(); + final LinkedList snapshotList = new LinkedList<>(); timer.scheduleAtFixedRate(new TimerTask() { @Override @@ -69,16 +90,24 @@ public class TransactionProducer { timer.scheduleAtFixedRate(new TimerTask() { private void printStats() { if (snapshotList.size() >= 10) { - Long[] begin = snapshotList.getFirst(); - Long[] end = snapshotList.getLast(); + Snapshot begin = snapshotList.getFirst(); + Snapshot end = snapshotList.getLast(); + + final long sendCount = (end.sendRequestSuccessCount - begin.sendRequestSuccessCount) + + (end.sendRequestFailedCount - begin.sendRequestFailedCount); + final long sendTps = (sendCount * 1000L) / (end.endTime - begin.endTime); + final double averageRT = (end.sendMessageTimeTotal - begin.sendMessageTimeTotal) / (double) (end.sendRequestSuccessCount - begin.sendRequestSuccessCount); - final long sendTps = - (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); - final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); + final long failCount = end.sendRequestFailedCount - begin.sendRequestFailedCount; + final long checkCount = end.checkCount - begin.checkCount; + final long unexpectedCheck = end.unexpectedCheckCount - begin.unexpectedCheckCount; + final long dupCheck = end.duplicatedCheck - begin.duplicatedCheck; System.out.printf( - "Send TPS: %d Max RT: %d Average RT: %7.3f Send Failed: %d Response Failed: %d transaction checkCount: %d %n", - sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4], end[6]); + "Send TPS:%5d Max RT:%5d AVG RT:%3.1f Send Failed: %d check: %d unexpectedCheck: %d duplicatedCheck: %d %n", + sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, failCount, checkCount, + unexpectedCheck, dupCheck); + statsBenchmark.getSendMessageMaxRT().set(0); } } @@ -92,45 +121,53 @@ public class TransactionProducer { } }, 10000, 10000); - final TransactionCheckListener transactionCheckListener = - new TransactionCheckListenerBImpl(ischeckffalse, statsBenchmark); + final TransactionListener transactionCheckListener = new TransactionListenerImpl(statsBenchmark, config); final TransactionMQProducer producer = new TransactionMQProducer("benchmark_transaction_producer"); producer.setInstanceName(Long.toString(System.currentTimeMillis())); - producer.setTransactionCheckListener(transactionCheckListener); + producer.setTransactionListener(transactionCheckListener); producer.setDefaultTopicQueueNums(1000); + if (commandLine.hasOption('n')) { + String ns = commandLine.getOptionValue('n'); + producer.setNamesrvAddr(ns); + } producer.start(); - final TransactionExecuterBImpl tranExecuter = new TransactionExecuterBImpl(ischeck); - - for (int i = 0; i < threadCount; i++) { + for (int i = 0; i < config.threadCount; i++) { sendThreadPool.execute(new Runnable() { @Override public void run() { while (true) { + boolean success = false; + final long beginTimestamp = System.currentTimeMillis(); try { - // Thread.sleep(1000); - final long beginTimestamp = System.currentTimeMillis(); SendResult sendResult = - producer.sendMessageInTransaction(msg, tranExecuter, null); - if (sendResult != null) { - statsBenchmark.getSendRequestSuccessCount().incrementAndGet(); - statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet(); - } - + producer.sendMessageInTransaction(buildMessage(config), null); + success = sendResult != null && sendResult.getSendStatus() == SendStatus.SEND_OK; + } catch (Throwable e) { + success = false; + } finally { final long currentRT = System.currentTimeMillis() - beginTimestamp; - statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT); + statsBenchmark.getSendMessageTimeTotal().addAndGet(currentRT); long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); while (currentRT > prevMaxRT) { - boolean updated = - statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, - currentRT); + boolean updated = statsBenchmark.getSendMessageMaxRT() + .compareAndSet(prevMaxRT, currentRT); if (updated) break; prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); } - } catch (MQClientException e) { - statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + if (success) { + statsBenchmark.getSendRequestSuccessCount().incrementAndGet(); + } else { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + } + if (config.sendInterval > 0) { + try { + Thread.sleep(config.sendInterval); + } catch (InterruptedException e) { + } + } } } } @@ -138,86 +175,222 @@ public class TransactionProducer { } } - private static Message buildMessage(final int messageSize) throws UnsupportedEncodingException { - Message msg = new Message(); - msg.setTopic("BenchmarkTest"); + private static Message buildMessage(TxSendConfig config) { + byte[] bs = new byte[config.messageSize]; + ThreadLocalRandom r = ThreadLocalRandom.current(); + r.nextBytes(bs); + + ByteBuffer buf = ByteBuffer.wrap(bs); + buf.putLong(config.batchId); + long sendMachineId = START_TIME << 32; + long msgId = sendMachineId | MSG_COUNT.getAndIncrement(); + buf.putLong(msgId); + + // save send tx result in message + if (r.nextDouble() < config.sendRollbackRate) { + buf.put((byte) LocalTransactionState.ROLLBACK_MESSAGE.ordinal()); + } else if (r.nextDouble() < config.sendUnknownRate) { + buf.put((byte) LocalTransactionState.UNKNOW.ordinal()); + } else { + buf.put((byte) LocalTransactionState.COMMIT_MESSAGE.ordinal()); + } - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < messageSize; i += 10) { - sb.append("hello baby"); + // save check tx result in message + for (int i = 0; i < MAX_CHECK_RESULT_IN_MSG; i++) { + if (r.nextDouble() < config.checkRollbackRate) { + buf.put((byte) LocalTransactionState.ROLLBACK_MESSAGE.ordinal()); + } else if (r.nextDouble() < config.checkUnknownRate) { + buf.put((byte) LocalTransactionState.UNKNOW.ordinal()); + } else { + buf.put((byte) LocalTransactionState.COMMIT_MESSAGE.ordinal()); + } } - msg.setBody(sb.toString().getBytes(RemotingHelper.DEFAULT_CHARSET)); + Message msg = new Message(); + msg.setTopic(config.topic); + msg.setBody(bs); return msg; } + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("w", "threadCount", true, "Thread count, Default: 32"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "messageSize", true, "Message Size, Default: 2048"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("t", "topic", true, "Topic name, Default: BenchmarkTest"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("sr", "send rollback rate", true, "Send rollback rate, Default: 0.0"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("su", "send unknown rate", true, "Send unknown rate, Default: 0.0"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("cr", "check rollback rate", true, "Check rollback rate, Default: 0.0"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("cu", "check unknown rate", true, "Check unknown rate, Default: 0.0"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("b", "test batch id", true, "test batch id, Default: System.currentMillis()"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("i", "send interval", true, "sleep interval in millis between messages, Default: 0"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } } -class TransactionExecuterBImpl implements LocalTransactionExecuter { +class TransactionListenerImpl implements TransactionListener { + private StatsBenchmarkTProducer statBenchmark; + private TxSendConfig sendConfig; + private final LRUMap cache = new LRUMap<>(200000); - private boolean ischeck; + private class MsgMeta { + long batchId; + long msgId; + LocalTransactionState sendResult; + List checkResult; + } - public TransactionExecuterBImpl(boolean ischeck) { - this.ischeck = ischeck; + public TransactionListenerImpl(StatsBenchmarkTProducer statsBenchmark, TxSendConfig sendConfig) { + this.statBenchmark = statsBenchmark; + this.sendConfig = sendConfig; } @Override - public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { - if (ischeck) { - return LocalTransactionState.UNKNOW; - } - return LocalTransactionState.COMMIT_MESSAGE; + public LocalTransactionState executeLocalTransaction(Message msg, Object arg) { + return parseFromMsg(msg).sendResult; } -} - -class TransactionCheckListenerBImpl implements TransactionCheckListener { - private boolean ischeckffalse; - private StatsBenchmarkTProducer statsBenchmarkTProducer; - public TransactionCheckListenerBImpl(boolean ischeckffalse, - StatsBenchmarkTProducer statsBenchmarkTProducer) { - this.ischeckffalse = ischeckffalse; - this.statsBenchmarkTProducer = statsBenchmarkTProducer; + private MsgMeta parseFromMsg(Message msg) { + byte[] bs = msg.getBody(); + ByteBuffer buf = ByteBuffer.wrap(bs); + MsgMeta msgMeta = new MsgMeta(); + msgMeta.batchId = buf.getLong(); + msgMeta.msgId = buf.getLong(); + msgMeta.sendResult = LocalTransactionState.values()[buf.get()]; + msgMeta.checkResult = new ArrayList<>(); + for (int i = 0; i < TransactionProducer.MAX_CHECK_RESULT_IN_MSG; i++) { + msgMeta.checkResult.add(LocalTransactionState.values()[buf.get()]); + } + return msgMeta; } @Override - public LocalTransactionState checkLocalTransactionState(MessageExt msg) { - statsBenchmarkTProducer.getCheckRequestSuccessCount().incrementAndGet(); - if (ischeckffalse) { + public LocalTransactionState checkLocalTransaction(MessageExt msg) { + MsgMeta msgMeta = parseFromMsg(msg); + if (msgMeta.batchId != sendConfig.batchId) { + // message not generated in this test + return LocalTransactionState.ROLLBACK_MESSAGE; + } + statBenchmark.getCheckCount().incrementAndGet(); + int times = 0; + try { + String checkTimes = msg.getUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES); + times = Integer.parseInt(checkTimes); + } catch (Exception e) { return LocalTransactionState.ROLLBACK_MESSAGE; } + times = times <= 0 ? 1 : times; + + boolean dup; + synchronized (cache) { + Integer oldCheckLog = cache.get(msgMeta.msgId); + Integer newCheckLog; + if (oldCheckLog == null) { + newCheckLog = 1 << (times - 1); + } else { + newCheckLog = oldCheckLog | (1 << (times - 1)); + } + dup = newCheckLog.equals(oldCheckLog); + } + if (dup) { + statBenchmark.getDuplicatedCheckCount().incrementAndGet(); + } + if (msgMeta.sendResult != LocalTransactionState.UNKNOW) { + System.out.printf("%s unexpected check: msgId=%s,txId=%s,checkTimes=%s,sendResult=%s\n", + new SimpleDateFormat("HH:mm:ss,SSS").format(new Date()), + msg.getMsgId(), msg.getTransactionId(), + msg.getUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES), + msgMeta.sendResult.toString()); + statBenchmark.getUnexpectedCheckCount().incrementAndGet(); + return msgMeta.sendResult; + } - return LocalTransactionState.COMMIT_MESSAGE; + for (int i = 0; i < times - 1; i++) { + LocalTransactionState s = msgMeta.checkResult.get(i); + if (s != LocalTransactionState.UNKNOW) { + System.out.printf("%s unexpected check: msgId=%s,txId=%s,checkTimes=%s,sendResult,lastCheckResult=%s\n", + new SimpleDateFormat("HH:mm:ss,SSS").format(new Date()), + msg.getMsgId(), msg.getTransactionId(), + msg.getUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES), s); + statBenchmark.getUnexpectedCheckCount().incrementAndGet(); + return s; + } + } + return msgMeta.checkResult.get(times - 1); } } +class Snapshot { + long endTime; + + long sendRequestSuccessCount; + + long sendRequestFailedCount; + + long sendMessageTimeTotal; + + long sendMessageMaxRT; + + long checkCount; + + long unexpectedCheckCount; + + long duplicatedCheck; +} + class StatsBenchmarkTProducer { private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L); private final AtomicLong sendRequestFailedCount = new AtomicLong(0L); - private final AtomicLong receiveResponseSuccessCount = new AtomicLong(0L); - - private final AtomicLong receiveResponseFailedCount = new AtomicLong(0L); - - private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L); + private final AtomicLong sendMessageTimeTotal = new AtomicLong(0L); private final AtomicLong sendMessageMaxRT = new AtomicLong(0L); - private final AtomicLong checkRequestSuccessCount = new AtomicLong(0L); + private final AtomicLong checkCount = new AtomicLong(0L); + + private final AtomicLong unexpectedCheckCount = new AtomicLong(0L); - public Long[] createSnapshot() { - Long[] snap = new Long[] { - System.currentTimeMillis(), - this.sendRequestSuccessCount.get(), - this.sendRequestFailedCount.get(), - this.receiveResponseSuccessCount.get(), - this.receiveResponseFailedCount.get(), - this.sendMessageSuccessTimeTotal.get(), - this.checkRequestSuccessCount.get()}; + private final AtomicLong duplicatedCheckCount = new AtomicLong(0); - return snap; + public Snapshot createSnapshot() { + Snapshot s = new Snapshot(); + s.endTime = System.currentTimeMillis(); + s.sendRequestSuccessCount = sendRequestSuccessCount.get(); + s.sendRequestFailedCount = sendRequestFailedCount.get(); + s.sendMessageTimeTotal = sendMessageTimeTotal.get(); + s.sendMessageMaxRT = sendMessageMaxRT.get(); + s.checkCount = checkCount.get(); + s.unexpectedCheckCount = unexpectedCheckCount.get(); + s.duplicatedCheck = duplicatedCheckCount.get(); + return s; } public AtomicLong getSendRequestSuccessCount() { @@ -228,23 +401,49 @@ class StatsBenchmarkTProducer { return sendRequestFailedCount; } - public AtomicLong getReceiveResponseSuccessCount() { - return receiveResponseSuccessCount; + public AtomicLong getSendMessageTimeTotal() { + return sendMessageTimeTotal; + } + + public AtomicLong getSendMessageMaxRT() { + return sendMessageMaxRT; } - public AtomicLong getReceiveResponseFailedCount() { - return receiveResponseFailedCount; + public AtomicLong getCheckCount() { + return checkCount; } - public AtomicLong getSendMessageSuccessTimeTotal() { - return sendMessageSuccessTimeTotal; + public AtomicLong getUnexpectedCheckCount() { + return unexpectedCheckCount; } - public AtomicLong getSendMessageMaxRT() { - return sendMessageMaxRT; + public AtomicLong getDuplicatedCheckCount() { + return duplicatedCheckCount; } +} + +class TxSendConfig { + String topic; + int threadCount; + int messageSize; + double sendRollbackRate; + double sendUnknownRate; + double checkRollbackRate; + double checkUnknownRate; + long batchId; + int sendInterval; +} + +class LRUMap extends LinkedHashMap { - public AtomicLong getCheckRequestSuccessCount() { - return checkRequestSuccessCount; + private int maxSize; + + public LRUMap(int maxSize) { + this.maxSize = maxSize; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > maxSize; } } diff --git a/example/src/main/java/org/apache/rocketmq/example/broadcast/PushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/broadcast/PushConsumer.java index fb1f9bbde7110fe0b9a500e771cf70a8fe4f0c95..28e02341c69b6ea96a16d08cb22b0b06480c8a82 100644 --- a/example/src/main/java/org/apache/rocketmq/example/broadcast/PushConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/broadcast/PushConsumer.java @@ -50,4 +50,4 @@ public class PushConsumer { consumer.start(); System.out.printf("Broadcast Consumer Started.%n"); } -} +} \ No newline at end of file diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java new file mode 100644 index 0000000000000000000000000000000000000000..072291d5c2ec8be0e7bd176f9d8fa3dd8d36b2ae --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.rpc; + +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.RequestCallback; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.common.RemotingHelper; + +public class AsyncRequestProducer { + private static final InternalLogger log = ClientLogger.getLog(); + + public static void main(String[] args) throws MQClientException, InterruptedException { + String producerGroup = "please_rename_unique_group_name"; + String topic = "RequestTopic"; + long ttl = 3000; + + DefaultMQProducer producer = new DefaultMQProducer(producerGroup); + producer.start(); + + try { + Message msg = new Message(topic, + "", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + + long begin = System.currentTimeMillis(); + producer.request(msg, new RequestCallback() { + @Override + public void onSuccess(Message message) { + long cost = System.currentTimeMillis() - begin; + System.out.printf("request to <%s> cost: %d replyMessage: %s %n", topic, cost, message); + } + + @Override + public void onException(Throwable e) { + System.err.printf("request to <%s> fail.", topic); + } + }, ttl); + } catch (Exception e) { + log.warn("", e); + } + /* shutdown after your request callback is finished */ +// producer.shutdown(); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java new file mode 100644 index 0000000000000000000000000000000000000000..b34908b84f34bfb10c7f16145d9a1fefd6295de6 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.rpc; + +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.remoting.common.RemotingHelper; + +public class RequestProducer { + public static void main(String[] args) throws MQClientException, InterruptedException { + String producerGroup = "please_rename_unique_group_name"; + String topic = "RequestTopic"; + long ttl = 3000; + + DefaultMQProducer producer = new DefaultMQProducer(producerGroup); + producer.start(); + + try { + Message msg = new Message(topic, + "", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + + long begin = System.currentTimeMillis(); + Message retMsg = producer.request(msg, ttl); + long cost = System.currentTimeMillis() - begin; + System.out.printf("request to <%s> cost: %d replyMessage: %s %n", topic, cost, retMsg); + } catch (Exception e) { + e.printStackTrace(); + } + producer.shutdown(); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java new file mode 100644 index 0000000000000000000000000000000000000000..c62c7d4eb6da90529b7bb7b208b71bb83cd2a8ca --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.rpc; + +import java.util.List; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.utils.MessageUtil; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.exception.RemotingException; + +public class ResponseConsumer { + public static void main(String[] args) throws InterruptedException, MQClientException { + String producerGroup = "please_rename_unique_group_name"; + String consumerGroup = "please_rename_unique_group_name"; + String topic = "RequestTopic"; + + // create a producer to send reply message + DefaultMQProducer replyProducer = new DefaultMQProducer(producerGroup); + replyProducer.start(); + + // create consumer + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroup); + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + + // recommend client configs + consumer.setPullTimeDelayMillsWhenException(0L); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); + for (MessageExt msg : msgs) { + try { + System.out.printf("handle message: %s", msg.toString()); + String replyTo = MessageUtil.getReplyToClient(msg); + byte[] replyContent = "reply message contents.".getBytes(); + // create reply message with given util, do not create reply message by yourself + Message replyMessage = MessageUtil.createReplyMessage(msg, replyContent); + + // send reply message with producer + SendResult replyResult = replyProducer.send(replyMessage, 3000); + System.out.printf("reply to %s , %s %n", replyTo, replyResult.toString()); + } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) { + e.printStackTrace(); + } + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.subscribe(topic, "*"); + consumer.start(); + System.out.printf("Consumer Started.%n"); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerAssign.java b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerAssign.java new file mode 100644 index 0000000000000000000000000000000000000000..e638de1c960ba9fd37a9f16515161d2aa3afe712 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerAssign.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.simple; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; + +public class LitePullConsumerAssign { + + public static volatile boolean running = true; + + public static void main(String[] args) throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer("please_rename_unique_group_name"); + litePullConsumer.setAutoCommit(false); + litePullConsumer.start(); + Collection mqSet = litePullConsumer.fetchMessageQueues("TopicTest"); + List list = new ArrayList<>(mqSet); + List assignList = new ArrayList<>(); + for (int i = 0; i < list.size() / 2; i++) { + assignList.add(list.get(i)); + } + litePullConsumer.assign(assignList); + litePullConsumer.seek(assignList.get(0), 10); + try { + while (running) { + List messageExts = litePullConsumer.poll(); + System.out.printf("%s %n", messageExts); + litePullConsumer.commitSync(); + } + } finally { + litePullConsumer.shutdown(); + } + + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java new file mode 100644 index 0000000000000000000000000000000000000000..e5c1a6134b4631e1e4dfb1dadf5ecccf1c530fd5 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.simple; + +import java.util.List; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageExt; + +public class LitePullConsumerSubscribe { + + public static volatile boolean running = true; + + public static void main(String[] args) throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer("lite_pull_consumer_test"); + litePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + litePullConsumer.subscribe("TopicTest", "*"); + litePullConsumer.start(); + try { + while (running) { + List messageExts = litePullConsumer.poll(); + System.out.printf("%s%n", messageExts); + } + } finally { + litePullConsumer.shutdown(); + } + } +} diff --git a/filter/pom.xml b/filter/pom.xml index cddf4d78638d1c1ff0d9718676ae979be782d473..4d02f1feed03f55a8028de920dc2203833624f13 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.5.2 + 4.6.1 4.0.0 diff --git a/logappender/pom.xml b/logappender/pom.xml index b82cb9360e127367e46d0be10d3fbf8a1d085a5c..9b102eddcc9cce19a4ed7775c3fdfbdfba433fad 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 rocketmq-logappender diff --git a/logging/pom.xml b/logging/pom.xml index 57f6cedbf8a93dc60db3437dccb6f50e8c6f3768..059869c860d250171a73f1d0335fa06055410bb1 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index c8de9290ce06de10c662fbf4b02af26195177b36..fe583a62c043d149cd05f5a1c5d7e63ec918e2d3 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessorTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessorTest.java index a0e8137524afe2488663fa33b8ac36f667ce4db6..c8bf057606d6507c1c0ef434d37a02a332488a1b 100644 --- a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessorTest.java +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessorTest.java @@ -52,7 +52,7 @@ import static org.mockito.Mockito.when; public class ClusterTestRequestProcessorTest { private ClusterTestRequestProcessor clusterTestProcessor; private DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private MQClientAPIImpl mQClientAPIImpl; private ChannelHandlerContext ctx; diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 8ed0c6d6fc2e54a58102df42bc6c40781729a605..66d7b23b996b7c715c8f1878a879c5bdf8ba5659 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.5.2 + 4.6.1 4.0.0 diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java index 93e60a738e94a6e041dd8f6f0c98749b3c5a5159..d2fc6781654a5ef7207a533de1835cb4de252f1b 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/LocalMessageCache.java @@ -73,7 +73,7 @@ class LocalMessageCache implements ServiceLifecycle { pullOffsetTable.putIfAbsent(remoteQueue, rocketmqPullConsumer.fetchConsumeOffset(remoteQueue, false)); } catch (MQClientException e) { - log.error("A error occurred in fetch consume offset process.", e); + log.error("An error occurred in fetch consume offset process.", e); } } return pullOffsetTable.get(remoteQueue); @@ -124,7 +124,7 @@ class LocalMessageCache implements ServiceLifecycle { try { rocketmqPullConsumer.updateConsumeOffset(consumeRequest.getMessageQueue(), offset); } catch (MQClientException e) { - log.error("A error occurred in update consume offset process.", e); + log.error("An error occurred in update consume offset process.", e); } } } @@ -135,7 +135,7 @@ class LocalMessageCache implements ServiceLifecycle { try { rocketmqPullConsumer.updateConsumeOffset(messageQueue, offset); } catch (MQClientException e) { - log.error("A error occurred in update consume offset process.", e); + log.error("An error occurred in update consume offset process.", e); } } diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java index d6735100ce2cb4a6496daff3d4ff55c8939657f8..945ecfac8551c670293598a6a53b04710a2472ed 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/consumer/PullConsumerImpl.java @@ -167,7 +167,7 @@ public class PullConsumerImpl implements PullConsumer { } localMessageCache.updatePullOffset(mq, pullResult.getNextBeginOffset()); } catch (Exception e) { - log.error("A error occurred in pull message process.", e); + log.error("An error occurred in pull message process.", e); } } }); diff --git a/pom.xml b/pom.xml index 0f9c311f568b0c05f62a49c260c89e0a87d47158..5d157f349de9955d7d80527153c0aafee2fdbd31 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 2012 org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -42,7 +42,7 @@ git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git - rocketmq-all-4.5.2 + rocketmq-all-4.6.1 @@ -106,6 +106,7 @@ ${project.basedir}/../test/target/jacoco-it.exec file:**/generated-sources/**,**/test/** + 2.0.2 @@ -458,6 +459,18 @@ 2.23.0 test + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + @@ -607,6 +620,13 @@ log4j-slf4j-impl 2.7 + + commons-validator + commons-validator + 1.6 + + + diff --git a/remoting/pom.xml b/remoting/pom.xml index 61b480c1c8a07664c3a35b02d11c86d84c4cd319..7fa4a76cf761c2d7c5a84bf39edf92dd26edfc7f 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java index 585b60b798cd0986748f9df035b6229690464f05..f244bf4c853551ad8b4810f95a78b039ae6d239b 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java @@ -17,18 +17,17 @@ package org.apache.rocketmq.remoting.common; import io.netty.channel.Channel; -import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; -import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.remoting.protocol.RemotingCommand; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.ByteBuffer; import java.nio.channels.SocketChannel; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; public class RemotingHelper { public static final String ROCKETMQ_REMOTING = "RocketmqRemoting"; @@ -53,8 +52,10 @@ public class RemotingHelper { } public static SocketAddress string2SocketAddress(final String addr) { - String[] s = addr.split(":"); - InetSocketAddress isa = new InetSocketAddress(s[0], Integer.parseInt(s[1])); + int split = addr.lastIndexOf(":"); + String host = addr.substring(0, split); + String port = addr.substring(split + 1); + InetSocketAddress isa = new InetSocketAddress(host, Integer.parseInt(port)); return isa; } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java index 3da3a1839603396a1bd592971059b436a1082e08..a16940e561c5cec23c9df021c429e1c64d500b88 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java @@ -31,7 +31,6 @@ import java.nio.channels.SocketChannel; import java.nio.channels.spi.SelectorProvider; import java.util.ArrayList; import java.util.Enumeration; - import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -145,8 +144,10 @@ public class RemotingUtil { } public static SocketAddress string2SocketAddress(final String addr) { - String[] s = addr.split(":"); - InetSocketAddress isa = new InetSocketAddress(s[0], Integer.parseInt(s[1])); + int split = addr.lastIndexOf(":"); + String host = addr.substring(0, split); + String port = addr.substring(split + 1); + InetSocketAddress isa = new InetSocketAddress(host, Integer.parseInt(port)); return isa; } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/exception/RemotingConnectException.java b/remoting/src/main/java/org/apache/rocketmq/remoting/exception/RemotingConnectException.java index 8286177c2efb15b6341cd22771f5a31339650b93..40e3b5aeefbe43db9421baf2dc143809dd9bc053 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/exception/RemotingConnectException.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/exception/RemotingConnectException.java @@ -24,6 +24,6 @@ public class RemotingConnectException extends RemotingException { } public RemotingConnectException(String addr, Throwable cause) { - super("connect to <" + addr + "> failed", cause); + super("connect to " + addr + " failed", cause); } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index fc9df37c652dcaef17b95a77bf2bffdaf60c9ba7..415d4956b25c5da5897bb9f25e644f0867078919 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -358,8 +358,6 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti } } - - @Override public RemotingCommand invokeSync(String addr, final RemotingCommand request, long timeoutMillis) throws InterruptedException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException { @@ -393,7 +391,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti } } - private Channel getAndCreateChannel(final String addr) throws InterruptedException { + private Channel getAndCreateChannel(final String addr) throws RemotingConnectException, InterruptedException { if (null == addr) { return getAndCreateNameserverChannel(); } @@ -406,7 +404,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti return this.createChannel(addr); } - private Channel getAndCreateNameserverChannel() throws InterruptedException { + private Channel getAndCreateNameserverChannel() throws RemotingConnectException, InterruptedException { String addr = this.namesrvAddrChoosed.get(); if (addr != null) { ChannelWrapper cw = this.channelTables.get(addr); @@ -440,9 +438,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti return channelNew; } } + throw new RemotingConnectException(addrList.toString()); } - } catch (Exception e) { - log.error("getAndCreateNameserverChannel: create name server channel exception", e); } finally { this.lockNamesrvChannel.unlock(); } @@ -456,8 +453,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti private Channel createChannel(final String addr) throws InterruptedException { ChannelWrapper cw = this.channelTables.get(addr); if (cw != null && cw.isOK()) { - cw.getChannel().close(); - channelTables.remove(addr); + return cw.getChannel(); } if (this.lockChannelTables.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { @@ -467,9 +463,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti if (cw != null) { if (cw.isOK()) { - cw.getChannel().close(); - this.channelTables.remove(addr); - createNewConnection = true; + return cw.getChannel(); } else if (!cw.getChannelFuture().isDone()) { createNewConnection = false; } else { @@ -587,7 +581,6 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti return channelEventListener; } - @Override public ExecutorService getCallbackExecutor() { return callbackExecutor != null ? callbackExecutor : publicExecutor; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index 32d169b2656016ebdf565891d3d10c16b1890c40..f94d5ba9ca59c4b4bda96aad560ed26b77412a0c 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -373,7 +373,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti switch (tlsMode) { case DISABLED: ctx.close(); - log.warn("Clients intend to establish a SSL connection while this server is running in SSL disabled mode"); + log.warn("Clients intend to establish an SSL connection while this server is running in SSL disabled mode"); break; case PERMISSIVE: case ENFORCING: @@ -384,7 +384,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti log.info("Handlers prepended to channel pipeline to establish SSL connection"); } else { ctx.close(); - log.error("Trying to establish a SSL connection but sslContext is null"); + log.error("Trying to establish an SSL connection but sslContext is null"); } break; diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 1daacd5244d3e814173ff90e45cddb784cfbfff4..b51b5b50ebd0e5d6169addefe1524b09cab3f9f1 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java index 066d36cedd981390ea816e6661b2ddc489168024..421eedb427305c0298109f0c9be36c1bab24d300 100644 --- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java +++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java @@ -49,10 +49,11 @@ public class ServerUtil { commandLine = parser.parse(options, args); if (commandLine.hasOption('h')) { hf.printHelp(appName, options, true); - return null; + System.exit(0); } } catch (ParseException e) { hf.printHelp(appName, options, true); + System.exit(1); } return commandLine; diff --git a/store/pom.xml b/store/pom.xml index eaf17a2e63a75bdd8a6af3d25363a0783c0ca370..f9f45b4b232571223f2e49193f9afe27f079424b 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 2ab66e84634f119b7dcb3b08e95c3399f26aff44..352585cbb5fb91c62aff9cfc002de7fde81404d7 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -16,6 +16,8 @@ */ package org.apache.rocketmq.store; +import java.net.Inet6Address; +import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -26,14 +28,14 @@ import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.ha.HAService; @@ -270,11 +272,21 @@ public class CommitLog { long bornTimeStamp = byteBuffer.getLong(); - ByteBuffer byteBuffer1 = byteBuffer.get(bytesContent, 0, 8); + ByteBuffer byteBuffer1; + if ((sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0) { + byteBuffer1 = byteBuffer.get(bytesContent, 0, 4 + 4); + } else { + byteBuffer1 = byteBuffer.get(bytesContent, 0, 16 + 4); + } long storeTimestamp = byteBuffer.getLong(); - ByteBuffer byteBuffer2 = byteBuffer.get(bytesContent, 0, 8); + ByteBuffer byteBuffer2; + if ((sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0) { + byteBuffer2 = byteBuffer.get(bytesContent, 0, 4 + 4); + } else { + byteBuffer2 = byteBuffer.get(bytesContent, 0, 16 + 4); + } int reconsumeTimes = byteBuffer.getInt(); @@ -339,7 +351,7 @@ public class CommitLog { } } - int readLength = calMsgLength(bodyLen, topicLen, propertiesLength); + int readLength = calMsgLength(sysFlag, bodyLen, topicLen, propertiesLength); if (totalSize != readLength) { doNothingForDeadCode(reconsumeTimes); doNothingForDeadCode(flag); @@ -372,7 +384,9 @@ public class CommitLog { return new DispatchRequest(-1, false /* success */); } - protected static int calMsgLength(int bodyLength, int topicLength, int propertiesLength) { + protected static int calMsgLength(int sysFlag, int bodyLength, int topicLength, int propertiesLength) { + int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; + int storehostAddressLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 8 : 20; final int msgLen = 4 //TOTALSIZE + 4 //MAGICCODE + 4 //BODYCRC @@ -382,9 +396,9 @@ public class CommitLog { + 8 //PHYSICALOFFSET + 4 //SYSFLAG + 8 //BORNTIMESTAMP - + 8 //BORNHOST + + bornhostLength //BORNHOST + 8 //STORETIMESTAMP - + 8 //STOREHOSTADDRESS + + storehostAddressLength //STOREHOSTADDRESS + 4 //RECONSUMETIMES + 8 //Prepared Transaction Offset + 4 + (bodyLength > 0 ? bodyLength : 0) //BODY @@ -496,7 +510,10 @@ public class CommitLog { return false; } - long storeTimestamp = byteBuffer.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSTION); + int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION); + int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; + int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornhostLength; + long storeTimestamp = byteBuffer.getLong(msgStoreTimePos); if (0 == storeTimestamp) { return false; } @@ -569,7 +586,18 @@ public class CommitLog { } } + InetSocketAddress bornSocketAddress = (InetSocketAddress) msg.getBornHost(); + if (bornSocketAddress.getAddress() instanceof Inet6Address) { + msg.setBornHostV6Flag(); + } + + InetSocketAddress storeSocketAddress = (InetSocketAddress) msg.getStoreHost(); + if (storeSocketAddress.getAddress() instanceof Inet6Address) { + msg.setStoreHostAddressV6Flag(); + } + long elapsedTimeInLock = 0; + MappedFile unlockMappedFile = null; MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); @@ -714,6 +742,16 @@ public class CommitLog { return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); } + InetSocketAddress bornSocketAddress = (InetSocketAddress) messageExtBatch.getBornHost(); + if (bornSocketAddress.getAddress() instanceof Inet6Address) { + messageExtBatch.setBornHostV6Flag(); + } + + InetSocketAddress storeSocketAddress = (InetSocketAddress) messageExtBatch.getStoreHost(); + if (storeSocketAddress.getAddress() instanceof Inet6Address) { + messageExtBatch.setStoreHostAddressV6Flag(); + } + long elapsedTimeInLock = 0; MappedFile unlockMappedFile = null; MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); @@ -804,7 +842,10 @@ public class CommitLog { SelectMappedBufferResult result = this.getMessage(offset, size); if (null != result) { try { - return result.getByteBuffer().getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSTION); + int sysFlag = result.getByteBuffer().getInt(MessageDecoder.SYSFLAG_POSITION); + int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; + int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornhostLength; + return result.getByteBuffer().getLong(msgStoreTimePos); } finally { result.release(); } @@ -1170,6 +1211,7 @@ public class CommitLog { // File at the end of the minimum fixed length empty private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4; private final ByteBuffer msgIdMemory; + private final ByteBuffer msgIdV6Memory; // Store the message content private final ByteBuffer msgStoreItemMemory; // The maximum length of the message @@ -1179,10 +1221,9 @@ public class CommitLog { private final StringBuilder msgIdBuilder = new StringBuilder(); - private final ByteBuffer hostHolder = ByteBuffer.allocate(8); - DefaultAppendMessageCallback(final int size) { - this.msgIdMemory = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + this.msgIdMemory = ByteBuffer.allocate(4 + 4 + 8); + this.msgIdV6Memory = ByteBuffer.allocate(16 + 4 + 8); this.msgStoreItemMemory = ByteBuffer.allocate(size + END_FILE_MIN_BLANK_LENGTH); this.maxMessageSize = size; } @@ -1198,8 +1239,20 @@ public class CommitLog { // PHY OFFSET long wroteOffset = fileFromOffset + byteBuffer.position(); - this.resetByteBuffer(hostHolder, 8); - String msgId = MessageDecoder.createMessageId(this.msgIdMemory, msgInner.getStoreHostBytes(hostHolder), wroteOffset); + int sysflag = msgInner.getSysFlag(); + + int bornHostLength = (sysflag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; + int storeHostLength = (sysflag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; + ByteBuffer bornHostHolder = ByteBuffer.allocate(bornHostLength); + ByteBuffer storeHostHolder = ByteBuffer.allocate(storeHostLength); + + this.resetByteBuffer(storeHostHolder, storeHostLength); + String msgId; + if ((sysflag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0) { + msgId = MessageDecoder.createMessageId(this.msgIdMemory, msgInner.getStoreHostBytes(storeHostHolder), wroteOffset); + } else { + msgId = MessageDecoder.createMessageId(this.msgIdV6Memory, msgInner.getStoreHostBytes(storeHostHolder), wroteOffset); + } // Record ConsumeQueue information keyBuilder.setLength(0); @@ -1246,7 +1299,7 @@ public class CommitLog { final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length; - final int msgLen = calMsgLength(bodyLength, topicLength, propertiesLength); + final int msgLen = calMsgLength(msgInner.getSysFlag(), bodyLength, topicLength, propertiesLength); // Exceeds the maximum message if (msgLen > this.maxMessageSize) { @@ -1291,14 +1344,13 @@ public class CommitLog { // 9 BORNTIMESTAMP this.msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); // 10 BORNHOST - this.resetByteBuffer(hostHolder, 8); - this.msgStoreItemMemory.put(msgInner.getBornHostBytes(hostHolder)); + this.resetByteBuffer(bornHostHolder, bornHostLength); + this.msgStoreItemMemory.put(msgInner.getBornHostBytes(bornHostHolder)); // 11 STORETIMESTAMP this.msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); // 12 STOREHOSTADDRESS - this.resetByteBuffer(hostHolder, 8); - this.msgStoreItemMemory.put(msgInner.getStoreHostBytes(hostHolder)); - //this.msgBatchMemory.put(msgInner.getStoreHostBytes()); + this.resetByteBuffer(storeHostHolder, storeHostLength); + this.msgStoreItemMemory.put(msgInner.getStoreHostBytes(storeHostHolder)); // 13 RECONSUMETIMES this.msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); // 14 Prepared Transaction Offset @@ -1359,8 +1411,13 @@ public class CommitLog { msgIdBuilder.setLength(0); final long beginTimeMills = CommitLog.this.defaultMessageStore.now(); ByteBuffer messagesByteBuff = messageExtBatch.getEncodedBuff(); - this.resetByteBuffer(hostHolder, 8); - ByteBuffer storeHostBytes = messageExtBatch.getStoreHostBytes(hostHolder); + + int sysFlag = messageExtBatch.getSysFlag(); + int storeHostLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; + ByteBuffer storeHostHolder = ByteBuffer.allocate(storeHostLength); + + this.resetByteBuffer(storeHostHolder, storeHostLength); + ByteBuffer storeHostBytes = messageExtBatch.getStoreHostBytes(storeHostHolder); messagesByteBuff.mark(); while (messagesByteBuff.hasRemaining()) { // 1 TOTALSIZE @@ -1396,7 +1453,13 @@ public class CommitLog { messagesByteBuff.putLong(wroteOffset + totalMsgLen - msgLen); storeHostBytes.rewind(); - String msgId = MessageDecoder.createMessageId(this.msgIdMemory, storeHostBytes, wroteOffset + totalMsgLen - msgLen); + String msgId; + if ((sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0) { + msgId = MessageDecoder.createMessageId(this.msgIdMemory, storeHostBytes, wroteOffset + totalMsgLen - msgLen); + } else { + msgId = MessageDecoder.createMessageId(this.msgIdV6Memory, storeHostBytes, wroteOffset + totalMsgLen - msgLen); + } + if (msgIdBuilder.length() > 0) { msgIdBuilder.append(',').append(msgId); } else { @@ -1432,8 +1495,6 @@ public class CommitLog { // The maximum length of the message private final int maxMessageSize; - private final ByteBuffer hostHolder = ByteBuffer.allocate(8); - MessageExtBatchEncoder(final int size) { this.msgBatchMemory = ByteBuffer.allocateDirect(size); this.maxMessageSize = size; @@ -1443,6 +1504,13 @@ public class CommitLog { msgBatchMemory.clear(); //not thread-safe int totalMsgLen = 0; ByteBuffer messagesByteBuff = messageExtBatch.wrap(); + + int sysFlag = messageExtBatch.getSysFlag(); + int bornHostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; + int storeHostLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; + ByteBuffer bornHostHolder = ByteBuffer.allocate(bornHostLength); + ByteBuffer storeHostHolder = ByteBuffer.allocate(storeHostLength); + while (messagesByteBuff.hasRemaining()) { // 1 TOTALSIZE messagesByteBuff.getInt(); @@ -1466,7 +1534,7 @@ public class CommitLog { final int topicLength = topicData.length; - final int msgLen = calMsgLength(bodyLen, topicLength, propertiesLen); + final int msgLen = calMsgLength(messageExtBatch.getSysFlag(), bodyLen, topicLength, propertiesLen); // Exceeds the maximum message if (msgLen > this.maxMessageSize) { @@ -1500,13 +1568,13 @@ public class CommitLog { // 9 BORNTIMESTAMP this.msgBatchMemory.putLong(messageExtBatch.getBornTimestamp()); // 10 BORNHOST - this.resetByteBuffer(hostHolder, 8); - this.msgBatchMemory.put(messageExtBatch.getBornHostBytes(hostHolder)); + this.resetByteBuffer(bornHostHolder, bornHostLength); + this.msgBatchMemory.put(messageExtBatch.getBornHostBytes(bornHostHolder)); // 11 STORETIMESTAMP this.msgBatchMemory.putLong(messageExtBatch.getStoreTimestamp()); // 12 STOREHOSTADDRESS - this.resetByteBuffer(hostHolder, 8); - this.msgBatchMemory.put(messageExtBatch.getStoreHostBytes(hostHolder)); + this.resetByteBuffer(storeHostHolder, storeHostLength); + this.msgBatchMemory.put(messageExtBatch.getStoreHostBytes(storeHostHolder)); // 13 RECONSUMETIMES this.msgBatchMemory.putInt(messageExtBatch.getReconsumeTimes()); // 14 Prepared Transaction Offset, batch does not support transaction diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index a63d3746b30663e562778201f6c2ee4dd1365b17..87ff0a096cf57a7f0d1ea7a82a4f3e94a262e374 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.StorePathConfigHelper; public class ConsumeQueue { @@ -397,6 +398,10 @@ public class ConsumeQueue { boolean result = this.putMessagePositionInfo(request.getCommitLogOffset(), request.getMsgSize(), tagsCode, request.getConsumeQueueOffset()); if (result) { + if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE || + this.defaultMessageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) { + this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp(request.getStoreTimestamp()); + } this.defaultMessageStore.getStoreCheckpoint().setLogicsMsgTimestamp(request.getStoreTimestamp()); return; } else { diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 971b1e75b38da514fd7e8c5f12e5ff5e3dc9c2ab..d5ba5692a928e4b65b41ec94be0356b0c81dd1c9 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -19,6 +19,8 @@ package org.apache.rocketmq.store; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import java.net.Inet6Address; +import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.ByteBuffer; import java.nio.channels.FileLock; @@ -285,8 +287,6 @@ public class DefaultMessageStore implements MessageStore { this.shutdown = false; } - - public void shutdown() { if (!this.shutdown) { this.shutdown = true; @@ -469,7 +469,7 @@ public class DefaultMessageStore implements MessageStore { long diff = this.systemClock.now() - begin; return diff < 10000000 - && diff > this.messageStoreConfig.getOsPageCacheBusyTimeOutMills(); + && diff > this.messageStoreConfig.getOsPageCacheBusyTimeOutMills(); } @Override @@ -1042,7 +1042,9 @@ public class DefaultMessageStore implements MessageStore { int i = 0; for (; i < bufferConsumeQueue.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) { long offsetPy = bufferConsumeQueue.getByteBuffer().getLong(); - final ByteBuffer msgIdMemory = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + InetSocketAddress inetSocketAddress = (InetSocketAddress) storeHost; + int msgIdLength = (inetSocketAddress.getAddress() instanceof Inet6Address) ? 16 + 4 + 8 : 4 + 4 + 8; + final ByteBuffer msgIdMemory = ByteBuffer.allocate(msgIdLength); String msgId = MessageDecoder.createMessageId(msgIdMemory, MessageExt.socketAddress2ByteBuffer(storeHost), offsetPy); messageIds.put(msgId, nextOffset++); diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 5192e86defba1da4f0bb91080afec8fb3af5424c..13da48bfe9f08408c57212ea506b674939bbb3ac 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -68,12 +68,11 @@ public class DLedgerCommitLog extends CommitLog { //This offset separate the old commitlog from dledger commitlog private long dividedCommitlogOffset = -1; - private boolean isInrecoveringOldCommitlog = false; public DLedgerCommitLog(final DefaultMessageStore defaultMessageStore) { super(defaultMessageStore); - dLedgerConfig = new DLedgerConfig(); + dLedgerConfig = new DLedgerConfig(); dLedgerConfig.setEnableDiskForceClean(defaultMessageStore.getMessageStoreConfig().isCleanFileForciblyEnable()); dLedgerConfig.setStoreType(DLedgerConfig.FILE); dLedgerConfig.setSelfId(defaultMessageStore.getMessageStoreConfig().getdLegerSelfId()); @@ -158,8 +157,6 @@ public class DLedgerCommitLog extends CommitLog { log.warn("Should not set confirm offset {} for dleger commitlog", phyOffset); } - - @Override public long remainHowManyDataToCommit() { return dLedgerFileList.remainHowManyDataToCommit(); @@ -180,7 +177,7 @@ public class DLedgerCommitLog extends CommitLog { if (mappedFileQueue.getMappedFiles().isEmpty()) { refreshConfig(); //To prevent too much log in defaultMessageStore - return Integer.MAX_VALUE; + return Integer.MAX_VALUE; } else { disableDeleteDledger(); } @@ -201,7 +198,6 @@ public class DLedgerCommitLog extends CommitLog { return 1; } - public SelectMappedBufferResult convertSbr(SelectMmapBufferResult sbr) { if (sbr == null) { return null; @@ -232,7 +228,6 @@ public class DLedgerCommitLog extends CommitLog { return this.getData(offset, offset == 0); } - @Override public SelectMappedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) { if (offset < dividedCommitlogOffset) { @@ -246,7 +241,7 @@ public class DLedgerCommitLog extends CommitLog { if (mappedFile != null) { int pos = (int) (offset % mappedFileSize); SelectMmapBufferResult sbr = mappedFile.selectMappedBuffer(pos); - return convertSbr(truncate(sbr)); + return convertSbr(truncate(sbr)); } return null; @@ -278,7 +273,7 @@ public class DLedgerCommitLog extends CommitLog { if (mappedFile == null) { return; } - ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); byteBuffer.position(mappedFile.getWrotePosition()); boolean needWriteMagicCode = true; // 1 TOTAL SIZE @@ -311,7 +306,7 @@ public class DLedgerCommitLog extends CommitLog { } @Override - public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { + public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { recover(maxPhyOffsetOfConsumeQueue); } @@ -329,9 +324,9 @@ public class DLedgerCommitLog extends CommitLog { try { int bodyOffset = DLedgerEntry.BODY_OFFSET; int pos = byteBuffer.position(); - int magic = byteBuffer.getInt(); + int magic = byteBuffer.getInt(); //In dledger, this field is size, it must be gt 0, so it could prevent collision - int magicOld = byteBuffer.getInt(); + int magicOld = byteBuffer.getInt(); if (magicOld == CommitLog.BLANK_MAGIC_CODE || magicOld == CommitLog.MESSAGE_MAGIC_CODE) { byteBuffer.position(pos); return super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody); @@ -409,10 +404,10 @@ public class DLedgerCommitLog extends CommitLog { long elapsedTimeInLock; long queueOffset; try { - beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); + beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); encodeResult = this.messageSerializer.serialize(msg); queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey); - if (encodeResult.status != AppendMessageStatus.PUT_OK) { + if (encodeResult.status != AppendMessageStatus.PUT_OK) { return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)); } AppendEntryRequest request = new AppendEntryRequest(); @@ -423,8 +418,11 @@ public class DLedgerCommitLog extends CommitLog { if (dledgerFuture.getPos() == -1) { return new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)); } - long wroteOffset = dledgerFuture.getPos() + DLedgerEntry.BODY_OFFSET; - ByteBuffer buffer = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + long wroteOffset = dledgerFuture.getPos() + DLedgerEntry.BODY_OFFSET; + + int msgIdLength = (msg.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8; + ByteBuffer buffer = ByteBuffer.allocate(msgIdLength); + String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.data.length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock); @@ -491,8 +489,6 @@ public class DLedgerCommitLog extends CommitLog { return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); } - - @Override public SelectMappedBufferResult getMessage(final long offset, final int size) { if (offset < dividedCommitlogOffset) { @@ -502,7 +498,7 @@ public class DLedgerCommitLog extends CommitLog { MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, offset == 0); if (mappedFile != null) { int pos = (int) (offset % mappedFileSize); - return convertSbr(mappedFile.selectMappedBuffer(pos, size)); + return convertSbr(mappedFile.selectMappedBuffer(pos, size)); } return null; } @@ -559,6 +555,7 @@ public class DLedgerCommitLog extends CommitLog { private String queueOffsetKey; private byte[] data; private AppendMessageStatus status; + public EncodeResult(AppendMessageStatus status, byte[] data, String queueOffsetKey) { this.data = data; this.status = status; @@ -570,6 +567,7 @@ public class DLedgerCommitLog extends CommitLog { // File at the end of the minimum fixed length empty private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4; private final ByteBuffer msgIdMemory; + private final ByteBuffer msgIdV6Memory; // Store the message content private final ByteBuffer msgStoreItemMemory; // The maximum length of the message @@ -579,10 +577,11 @@ public class DLedgerCommitLog extends CommitLog { private final StringBuilder msgIdBuilder = new StringBuilder(); - private final ByteBuffer hostHolder = ByteBuffer.allocate(8); +// private final ByteBuffer hostHolder = ByteBuffer.allocate(8); MessageSerializer(final int size) { - this.msgIdMemory = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + this.msgIdMemory = ByteBuffer.allocate(4 + 4 + 8); + this.msgIdV6Memory = ByteBuffer.allocate(16 + 4 + 8); this.msgStoreItemMemory = ByteBuffer.allocate(size + END_FILE_MIN_BLANK_LENGTH); this.maxMessageSize = size; } @@ -597,7 +596,13 @@ public class DLedgerCommitLog extends CommitLog { // PHY OFFSET long wroteOffset = 0; - this.resetByteBuffer(hostHolder, 8); + int sysflag = msgInner.getSysFlag(); + + int bornHostLength = (sysflag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; + int storeHostLength = (sysflag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; + ByteBuffer bornHostHolder = ByteBuffer.allocate(bornHostLength); + ByteBuffer storeHostHolder = ByteBuffer.allocate(storeHostLength); + // Record ConsumeQueue information keyBuilder.setLength(0); keyBuilder.append(msgInner.getTopic()); @@ -644,7 +649,7 @@ public class DLedgerCommitLog extends CommitLog { final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length; - final int msgLen = calMsgLength(bodyLength, topicLength, propertiesLength); + final int msgLen = calMsgLength(msgInner.getSysFlag(), bodyLength, topicLength, propertiesLength); // Exceeds the maximum message if (msgLen > this.maxMessageSize) { @@ -673,13 +678,13 @@ public class DLedgerCommitLog extends CommitLog { // 9 BORNTIMESTAMP this.msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); // 10 BORNHOST - this.resetByteBuffer(hostHolder, 8); - this.msgStoreItemMemory.put(msgInner.getBornHostBytes(hostHolder)); + this.resetByteBuffer(bornHostHolder, bornHostLength); + this.msgStoreItemMemory.put(msgInner.getBornHostBytes(bornHostHolder)); // 11 STORETIMESTAMP this.msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); // 12 STOREHOSTADDRESS - this.resetByteBuffer(hostHolder, 8); - this.msgStoreItemMemory.put(msgInner.getStoreHostBytes(hostHolder)); + this.resetByteBuffer(storeHostHolder, storeHostLength); + this.msgStoreItemMemory.put(msgInner.getStoreHostBytes(storeHostHolder)); //this.msgBatchMemory.put(msgInner.getStoreHostBytes()); // 13 RECONSUMETIMES this.msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); @@ -714,6 +719,7 @@ public class DLedgerCommitLog extends CommitLog { public static class DLedgerSelectMappedBufferResult extends SelectMappedBufferResult { private SelectMmapBufferResult sbr; + public DLedgerSelectMappedBufferResult(SelectMmapBufferResult sbr) { super(sbr.getStartOffset(), sbr.getByteBuffer(), sbr.getSize(), null); this.sbr = sbr; diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java index 84fb4217eaaff387c7e759b54d28c3abeda1a2d0..3035c575cf1ccb644430118fb2bb459050291242 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java @@ -280,7 +280,9 @@ public class HAService { if (!this.requestsRead.isEmpty()) { for (CommitLog.GroupCommitRequest req : this.requestsRead) { boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); - for (int i = 0; !transferOK && i < 5; i++) { + long waitUntilWhen = HAService.this.defaultMessageStore.getSystemClock().now() + + HAService.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout(); + while (!transferOK && HAService.this.defaultMessageStore.getSystemClock().now() < waitUntilWhen) { this.notifyTransferObject.waitForRunning(1000); transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); } diff --git a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java index 50a48d4de9c26be4144a9fc938a76a5e6dcef096..3be8cbc809ed59f6d39de94cd407a01741a899be 100644 --- a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java +++ b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java @@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.common.ConfigManager; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; @@ -305,6 +306,11 @@ public class ScheduleMessageService extends ConfigManager { if (msgExt != null) { try { MessageExtBrokerInner msgInner = this.messageTimeup(msgExt); + if (MixAll.RMQ_SYS_TRANS_HALF_TOPIC.equals(msgInner.getTopic())) { + log.error("[BUG] the real topic of schedule msg is {}, discard the msg. msg={}", + msgInner.getTopic(), msgInner); + continue; + } PutMessageResult putMessageResult = ScheduleMessageService.this.writeMessageStore .putMessage(msgInner); diff --git a/store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java b/store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java index fbb2a746c4f96686b273adb4e0847f927a6cc1ff..f46b3befed63fc25f9c0901b60f97c54a68616f8 100644 --- a/store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/AppendCallbackTest.java @@ -97,6 +97,43 @@ public class AppendCallbackTest { assertTrue(result.getMsgId().length() > 0); //should have already constructed some message ids } + @Test + public void testAppendIPv6HostMessageBatchEndOfFile() throws Exception { + List messages = new ArrayList<>(); + String topic = "test-topic"; + int queue = 0; + for (int i = 0; i < 10; i++) { + Message msg = new Message(); + msg.setBody("body".getBytes()); + msg.setTopic(topic); + msg.setTags("abc"); + messages.add(msg); + } + MessageExtBatch messageExtBatch = new MessageExtBatch(); + messageExtBatch.setTopic(topic); + messageExtBatch.setQueueId(queue); + messageExtBatch.setBornTimestamp(System.currentTimeMillis()); + messageExtBatch.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + messageExtBatch.setSysFlag(0); + messageExtBatch.setBornHostV6Flag(); + messageExtBatch.setStoreHostAddressV6Flag(); + messageExtBatch.setBornHost(new InetSocketAddress("1050:0000:0000:0000:0005:0600:300c:326b", 123)); + messageExtBatch.setStoreHost(new InetSocketAddress("::1", 124)); + messageExtBatch.setBody(MessageDecoder.encodeMessages(messages)); + + messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch)); + ByteBuffer buff = ByteBuffer.allocate(1024 * 10); + //encounter end of file when append half of the data + AppendMessageResult result = callback.doAppend(0, buff, 1000, messageExtBatch); + assertEquals(AppendMessageStatus.END_OF_FILE, result.getStatus()); + assertEquals(0, result.getWroteOffset()); + assertEquals(0, result.getLogicsOffset()); + assertEquals(1000, result.getWroteBytes()); + assertEquals(8, buff.position()); //write blank size and magic value + + assertTrue(result.getMsgId().length() > 0); //should have already constructed some message ids + } + @Test public void testAppendMessageBatchSucc() throws Exception { List messages = new ArrayList<>(); @@ -153,4 +190,64 @@ public class AppendCallbackTest { } + @Test + public void testAppendIPv6HostMessageBatchSucc() throws Exception { + List messages = new ArrayList<>(); + String topic = "test-topic"; + int queue = 0; + for (int i = 0; i < 10; i++) { + Message msg = new Message(); + msg.setBody("body".getBytes()); + msg.setTopic(topic); + msg.setTags("abc"); + messages.add(msg); + } + MessageExtBatch messageExtBatch = new MessageExtBatch(); + messageExtBatch.setTopic(topic); + messageExtBatch.setQueueId(queue); + messageExtBatch.setBornTimestamp(System.currentTimeMillis()); + messageExtBatch.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + messageExtBatch.setSysFlag(0); + messageExtBatch.setBornHostV6Flag(); + messageExtBatch.setStoreHostAddressV6Flag(); + messageExtBatch.setBornHost(new InetSocketAddress("1050:0000:0000:0000:0005:0600:300c:326b", 123)); + messageExtBatch.setStoreHost(new InetSocketAddress("::1", 124)); + messageExtBatch.setBody(MessageDecoder.encodeMessages(messages)); + + messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch)); + ByteBuffer buff = ByteBuffer.allocate(1024 * 10); + AppendMessageResult allresult = callback.doAppend(0, buff, 1024 * 10, messageExtBatch); + + assertEquals(AppendMessageStatus.PUT_OK, allresult.getStatus()); + assertEquals(0, allresult.getWroteOffset()); + assertEquals(0, allresult.getLogicsOffset()); + assertEquals(buff.position(), allresult.getWroteBytes()); + + assertEquals(messages.size(), allresult.getMsgNum()); + + Set msgIds = new HashSet<>(); + for (String msgId : allresult.getMsgId().split(",")) { + assertEquals(56, msgId.length()); + msgIds.add(msgId); + } + assertEquals(messages.size(), msgIds.size()); + + List decodeMsgs = MessageDecoder.decodes((ByteBuffer) buff.flip()); + assertEquals(decodeMsgs.size(), decodeMsgs.size()); + long queueOffset = decodeMsgs.get(0).getQueueOffset(); + long storeTimeStamp = decodeMsgs.get(0).getStoreTimestamp(); + for (int i = 0; i < messages.size(); i++) { + assertEquals(messages.get(i).getTopic(), decodeMsgs.get(i).getTopic()); + assertEquals(new String(messages.get(i).getBody()), new String(decodeMsgs.get(i).getBody())); + assertEquals(messages.get(i).getTags(), decodeMsgs.get(i).getTags()); + + assertEquals(messageExtBatch.getBornHostNameString(), decodeMsgs.get(i).getBornHostNameString()); + + assertEquals(messageExtBatch.getBornTimestamp(), decodeMsgs.get(i).getBornTimestamp()); + assertEquals(storeTimeStamp, decodeMsgs.get(i).getStoreTimestamp()); + assertEquals(queueOffset++, decodeMsgs.get(i).getQueueOffset()); + } + + } + } diff --git a/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java b/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java index 0c8e5bb1472caf268cea6522de7ebf332a5db42c..8618dbb0e56ed2804303d1b08bbe94cc5e54f500 100644 --- a/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java @@ -23,6 +23,7 @@ import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.stats.BrokerStatsManager; @@ -125,6 +126,58 @@ public class BatchPutMessageTest { } + @Test + public void testPutIPv6HostMessages() throws Exception { + List messages = new ArrayList<>(); + String topic = "batch-write-topic"; + int queue = 0; + int[] msgLengthArr = new int[11]; + msgLengthArr[0] = 0; + int j = 1; + for (int i = 0; i < 10; i++) { + Message msg = new Message(); + msg.setBody(("body" + i).getBytes()); + msg.setTopic(topic); + msg.setTags("TAG1"); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + messages.add(msg); + String properties = messageProperties2String(msg.getProperties()); + byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8); + short propertiesLength = (short) propertiesBytes.length; + final byte[] topicData = msg.getTopic().getBytes(MessageDecoder.CHARSET_UTF8); + final int topicLength = topicData.length; + msgLengthArr[j] = calIPv6HostMsgLength(msg.getBody().length, topicLength, propertiesLength) + msgLengthArr[j - 1]; + j++; + } + byte[] batchMessageBody = MessageDecoder.encodeMessages(messages); + MessageExtBatch messageExtBatch = new MessageExtBatch(); + messageExtBatch.setTopic(topic); + messageExtBatch.setQueueId(queue); + messageExtBatch.setBody(batchMessageBody); + messageExtBatch.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + messageExtBatch.setBornTimestamp(System.currentTimeMillis()); + messageExtBatch.setSysFlag(0); + messageExtBatch.setBornHostV6Flag(); + messageExtBatch.setStoreHostAddressV6Flag(); + messageExtBatch.setStoreHost(new InetSocketAddress("1050:0000:0000:0000:0005:0600:300c:326b", 125)); + messageExtBatch.setBornHost(new InetSocketAddress("::1", 126)); + + PutMessageResult putMessageResult = messageStore.putMessages(messageExtBatch); + assertThat(putMessageResult.isOk()).isTrue(); + + Thread.sleep(3 * 1000); + + for (long i = 0; i < 10; i++) { + MessageExt messageExt = messageStore.lookMessageByOffset(msgLengthArr[(int) i]); + assertThat(messageExt).isNotNull(); + GetMessageResult result = messageStore.getMessage("batch_write_group", topic, queue, i, 1024 * 1024, null); + assertThat(result).isNotNull(); + assertThat(result.getStatus()).isEqualTo(GetMessageStatus.FOUND); + result.release(); + } + + } + private int calMsgLength(int bodyLength, int topicLength, int propertiesLength) { final int msgLen = 4 //TOTALSIZE + 4 //MAGICCODE @@ -147,6 +200,28 @@ public class BatchPutMessageTest { return msgLen; } + private int calIPv6HostMsgLength(int bodyLength, int topicLength, int propertiesLength) { + final int msgLen = 4 //TOTALSIZE + + 4 //MAGICCODE + + 4 //BODYCRC + + 4 //QUEUEID + + 4 //FLAG + + 8 //QUEUEOFFSET + + 8 //PHYSICALOFFSET + + 4 //SYSFLAG + + 8 //BORNTIMESTAMP + + 20 //BORNHOST + + 8 //STORETIMESTAMP + + 20 //STOREHOSTADDRESS + + 4 //RECONSUMETIMES + + 8 //Prepared Transaction Offset + + 4 + (bodyLength > 0 ? bodyLength : 0) //BODY + + 1 + topicLength //TOPIC + + 2 + (propertiesLength > 0 ? propertiesLength : 0) //propertiesLength + + 0; + return msgLen; + } + public String messageProperties2String(Map properties) { StringBuilder sb = new StringBuilder(); if (properties != null) { diff --git a/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java index c9730306b5d6793811191697062b9efc3f8894a6..7c57813da7e8bdc5753886a461856db383b30e06 100644 --- a/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java @@ -83,6 +83,29 @@ public class ConsumeQueueTest { return msg; } + public MessageExtBrokerInner buildIPv6HostMessage() { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic(topic); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(msgBody); + msg.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(queueId); + msg.setSysFlag(0); + msg.setBornHostV6Flag(); + msg.setStoreHostAddressV6Flag(); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setBornHost(new InetSocketAddress("1050:0000:0000:0000:0005:0600:300c:326b", 123)); + msg.setStoreHost(new InetSocketAddress("::1", 124)); + for (int i = 0; i < 1; i++) { + msg.putUserProperty(String.valueOf(i), "imagoodperson" + i); + } + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + + return msg; + } + public MessageStoreConfig buildStoreConfig(int commitLogFileSize, int cqFileSize, boolean enableCqExt, int cqExtFileSize) { MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); @@ -127,7 +150,11 @@ public class ConsumeQueueTest { long totalMsgs = 200; for (long i = 0; i < totalMsgs; i++) { - master.putMessage(buildMessage()); + if (i < totalMsgs / 2) { + master.putMessage(buildMessage()); + } else { + master.putMessage(buildIPv6HostMessage()); + } } } diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java index 785977468e05138198b8da07ee28d10c17c404ef..fd13e71a427d07db60400392a8718bcc64de0298 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java @@ -24,6 +24,7 @@ import java.lang.reflect.Method; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; @@ -116,13 +117,19 @@ public class DefaultMessageStoreTest { @Test public void testWriteAndRead() { - long totalMsgs = 10; + long ipv4HostMsgs = 10; + long ipv6HostMsgs = 10; + long totalMsgs = ipv4HostMsgs + ipv6HostMsgs; QUEUE_TOTAL = 1; MessageBody = StoreMessage.getBytes(); - for (long i = 0; i < totalMsgs; i++) { + for (long i = 0; i < ipv4HostMsgs; i++) { messageStore.putMessage(buildMessage()); } + for (long i = 0; i < ipv6HostMsgs; i++) { + messageStore.putMessage(buildIPv6HostMessage()); + } + StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore); for (long i = 0; i < totalMsgs; i++) { @@ -134,7 +141,7 @@ public class DefaultMessageStoreTest { } @Test - public void should_look_message_successfully_when_offset_is_first() { + public void testLookMessageByOffset_OffsetIsFirst() { final int totalCount = 10; int queueId = new Random().nextInt(10); String topic = "FooBar"; @@ -150,7 +157,7 @@ public class DefaultMessageStoreTest { } @Test - public void should_look_message_successfully_when_offset_is_last() { + public void testLookMessageByOffset_OffsetIsLast() { final int totalCount = 10; int queueId = new Random().nextInt(10); String topic = "FooBar"; @@ -164,7 +171,7 @@ public class DefaultMessageStoreTest { } @Test - public void should_look_message_failed_and_return_null_when_offset_is_out_of_bound() { + public void testLookMessageByOffset_OffsetIsOutOfBound() { final int totalCount = 10; int queueId = new Random().nextInt(10); String topic = "FooBar"; @@ -177,7 +184,7 @@ public class DefaultMessageStoreTest { } @Test - public void should_get_consume_queue_offset_successfully_when_incomming_by_timestamp() throws InterruptedException { + public void testGetOffsetInQueueByTime() { final int totalCount = 10; int queueId = 0; String topic = "FooBar"; @@ -196,7 +203,7 @@ public class DefaultMessageStoreTest { } @Test - public void should_get_consume_queue_offset_successfully_when_timestamp_is_skewing() throws InterruptedException { + public void testGetOffsetInQueueByTime_TimestampIsSkewing() { final int totalCount = 10; int queueId = 0; String topic = "FooBar"; @@ -221,7 +228,7 @@ public class DefaultMessageStoreTest { } @Test - public void should_get_min_of_max_consume_queue_offset_when_timestamp_s_skewing_is_large() throws InterruptedException { + public void testGetOffsetInQueueByTime_TimestampSkewingIsLarge() { final int totalCount = 10; int queueId = 0; String topic = "FooBar"; @@ -247,7 +254,7 @@ public class DefaultMessageStoreTest { } @Test - public void should_return_zero_when_consume_queue_not_found() throws InterruptedException { + public void testGetOffsetInQueueByTime_ConsumeQueueNotFound1() { final int totalCount = 10; int queueId = 0; int wrongQueueId = 1; @@ -263,7 +270,7 @@ public class DefaultMessageStoreTest { } @Test - public void should_return_negative_one_when_invoke_getMessageStoreTimeStamp_if_consume_queue_not_found() throws InterruptedException { + public void testGetOffsetInQueueByTime_ConsumeQueueNotFound2() { final int totalCount = 10; int queueId = 0; int wrongQueueId = 1; @@ -278,7 +285,7 @@ public class DefaultMessageStoreTest { } @Test - public void should_return_negative_one_when_invoke_getMessageStoreTimeStamp_if_consumeQueueOffset_not_exist() throws InterruptedException { + public void testGetOffsetInQueueByTime_ConsumeQueueOffsetNotExist() { final int totalCount = 10; int queueId = 0; int wrongQueueId = 1; @@ -293,9 +300,8 @@ public class DefaultMessageStoreTest { assertThat(messageStoreTimeStamp).isEqualTo(-1); } - @Test - public void should_get_message_store_timestamp_successfully_when_incomming_by_topic_queueId_and_consumeQueueOffset() throws InterruptedException { + public void testGetMessageStoreTimeStamp() { final int totalCount = 10; int queueId = 0; String topic = "FooBar"; @@ -304,7 +310,7 @@ public class DefaultMessageStoreTest { StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore); ConsumeQueue consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); - int minOffsetInQueue = (int)consumeQueue.getMinOffsetInQueue(); + int minOffsetInQueue = (int) consumeQueue.getMinOffsetInQueue(); for (int i = minOffsetInQueue; i < consumeQueue.getMaxOffsetInQueue(); i++) { long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, queueId, i); assertThat(messageStoreTimeStamp).isEqualTo(appendMessageResults[i].getStoreTimestamp()); @@ -312,14 +318,14 @@ public class DefaultMessageStoreTest { } @Test - public void should_return_negative_one_when_invoke_getStoreTime_if_incomming_param_is_null() { + public void testGetStoreTime_ParamIsNull() { long storeTime = getStoreTime(null); assertThat(storeTime).isEqualTo(-1); } @Test - public void should_get_store_time_successfully_when_invoke_getStoreTime_if_everything_is_ok() throws InterruptedException { + public void testGetStoreTime_EverythingIsOk() { final int totalCount = 10; int queueId = 0; String topic = "FooBar"; @@ -337,7 +343,7 @@ public class DefaultMessageStoreTest { } @Test - public void should_return_negative_one_when_invoke_getStoreTime_if_phyOffset_is_less_than_commitLog_s_minOffset() { + public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() { long phyOffset = -10; int size = 138; ByteBuffer byteBuffer = ByteBuffer.allocate(100); @@ -354,7 +360,7 @@ public class DefaultMessageStoreTest { } private DefaultMessageStore getDefaultMessageStore() { - return (DefaultMessageStore)this.messageStore; + return (DefaultMessageStore) this.messageStore; } private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId) { @@ -365,7 +371,9 @@ public class DefaultMessageStoreTest { AppendMessageResult[] appendMessageResultArray = new AppendMessageResult[totalCount]; for (int i = 0; i < totalCount; i++) { String messageBody = buildMessageBodyByOffset(StoreMessage, i); - MessageExtBrokerInner msgInner = buildMessage(messageBody.getBytes(), topic); + + MessageExtBrokerInner msgInner = + i < totalCount / 2 ? buildMessage(messageBody.getBytes(), topic) : buildIPv6HostMessage(messageBody.getBytes(), topic); msgInner.setQueueId(queueId); PutMessageResult result = messageStore.putMessage(msgInner); appendMessageResultArray[i] = result.getAppendMessageResult(); @@ -374,7 +382,7 @@ public class DefaultMessageStoreTest { try { Thread.sleep(10); } catch (InterruptedException e) { - throw new RuntimeException("Thread sleep ERROR"); + throw new RuntimeException("Thread sleep ERROR"); } } } @@ -397,7 +405,7 @@ public class DefaultMessageStoreTest { try { Method getStoreTime = getDefaultMessageStore().getClass().getDeclaredMethod("getStoreTime", SelectMappedBufferResult.class); getStoreTime.setAccessible(true); - return (long)getStoreTime.invoke(getDefaultMessageStore(), result); + return (long) getStoreTime.invoke(getDefaultMessageStore(), result); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { throw new RuntimeException(e); } @@ -418,10 +426,43 @@ public class DefaultMessageStoreTest { return msg; } + private MessageExtBrokerInner buildIPv6HostMessage(byte[] messageBody, String topic) { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic(topic); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(messageBody); + msg.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + msg.setSysFlag(0); + msg.setBornHostV6Flag(); + msg.setStoreHostAddressV6Flag(); + msg.setBornTimestamp(System.currentTimeMillis()); + try { + msg.setBornHost(new InetSocketAddress(InetAddress.getByName("1050:0000:0000:0000:0005:0600:300c:326b"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + + try { + msg.setStoreHost(new InetSocketAddress(InetAddress.getByName("::1"), 0)); + } catch (UnknownHostException e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + return msg; + } + private MessageExtBrokerInner buildMessage() { return buildMessage(MessageBody, "FooBar"); } + private MessageExtBrokerInner buildIPv6HostMessage() { + return buildIPv6HostMessage(MessageBody, "FooBar"); + } + private void verifyThatMasterIsFunctional(long totalMsgs, MessageStore master) { for (long i = 0; i < totalMsgs; i++) { master.putMessage(buildMessage()); @@ -477,7 +518,7 @@ public class DefaultMessageStoreTest { messageStore.putMessage(messageExtBrokerInner); } - // Thread.sleep(100);//wait for build consumer queue + // Thread.sleep(100);//wait for build consumer queue StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore); long maxPhyOffset = messageStore.getMaxPhyOffset(); @@ -587,7 +628,7 @@ public class DefaultMessageStoreTest { private class MyMessageArrivingListener implements MessageArrivingListener { @Override public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, - byte[] filterBitMap, Map properties) { + byte[] filterBitMap, Map properties) { } } } diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java index 0dcb3b254c0c949ad4f4186b8148dccb70332abf..a736754de47b4ce6d699b63360dfebf1eb86ee0d 100644 --- a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java @@ -20,12 +20,12 @@ import java.io.File; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.net.UnknownHostException; import java.util.HashSet; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.junit.After; public class StoreTestBase { @@ -59,6 +59,33 @@ public class StoreTestBase { return msg; } + protected MessageExtBrokerInner buildIPv6HostMessage() { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic("StoreTest"); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(MessageBody); + msg.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + msg.setSysFlag(0); + msg.setBornHostV6Flag(); + msg.setStoreHostAddressV6Flag(); + msg.setBornTimestamp(System.currentTimeMillis()); + try { + msg.setBornHost(new InetSocketAddress(InetAddress.getByName("1050:0000:0000:0000:0005:0600:300c:326b"), 8123)); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + + try { + msg.setStoreHost(new InetSocketAddress(InetAddress.getByName("::1"), 8123)); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + return msg; + } + public static String createBaseDir() { String baseDir = System.getProperty("user.home") + File.separator + "unitteststore" + File.separator + UUID.randomUUID(); final File file = new File(baseDir); @@ -74,7 +101,6 @@ public class StoreTestBase { return file.createNewFile(); } - public static void deleteFile(String fileName) { deleteFile(new File(fileName)); } diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index d0829d122a1ca1ad250053678f812eb1edac7c57..f0b9205302e965a832393d788d122e64bbb5d0aa 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -145,7 +145,8 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { List results = new ArrayList<>(); for (int i = 0; i < 10; i++) { - MessageExtBrokerInner msgInner = buildMessage(); + MessageExtBrokerInner msgInner = + i < 5 ? buildMessage() : buildIPv6HostMessage(); msgInner.setTopic(topic); msgInner.setQueueId(0); PutMessageResult putMessageResult = messageStore.putMessage(msgInner); @@ -209,5 +210,39 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { followerStore.shutdown(); } + @Test + public void testIPv6HostMsgCommittedPos() throws Exception { + String peers = String.format("n0-localhost:%d;n1-localhost:%d", nextPort(), nextPort()); + String group = UUID.randomUUID().toString(); + DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group,"n0", peers, "n0", false, 0); + + String topic = UUID.randomUUID().toString(); + MessageExtBrokerInner msgInner = buildIPv6HostMessage(); + msgInner.setTopic(topic); + msgInner.setQueueId(0); + PutMessageResult putMessageResult = leaderStore.putMessage(msgInner); + Assert.assertEquals(PutMessageStatus.OS_PAGECACHE_BUSY, putMessageResult.getPutMessageStatus()); + + Thread.sleep(1000); + + Assert.assertEquals(0, leaderStore.getCommitLog().getMaxOffset()); + Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0)); + + + DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group,"n1", peers, "n0", false, 0); + Thread.sleep(2000); + + Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(1, followerStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertTrue(leaderStore.getCommitLog().getMaxOffset() > 0); + + + leaderStore.destroy(); + followerStore.destroy(); + + leaderStore.shutdown(); + followerStore.shutdown(); + } + } diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java index 2da2fb7a2d5259a3fbbf5f4547607bee90e866f9..5b0ca347579dc7e391dab25645d5a87620a89269 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -19,6 +19,7 @@ package org.apache.rocketmq.store.dledger; import io.openmessaging.storage.dledger.DLedgerConfig; import io.openmessaging.storage.dledger.DLedgerServer; import java.io.File; +import java.net.UnknownHostException; import java.util.Arrays; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.message.MessageDecoder; @@ -118,7 +119,7 @@ public class MessageStoreTestBase extends StoreTestBase { return defaultMessageStore; } - protected void doPutMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) { + protected void doPutMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) throws UnknownHostException { for (int i = 0; i < num; i++) { MessageExtBrokerInner msgInner = buildMessage(); msgInner.setTopic(topic); diff --git a/test/pom.xml b/test/pom.xml index 52dffebe14539688b8d4bef4f69e1a62282741b1..54c08596a90efb889dd3f186a9281e5305dc904c 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.5.2 + 4.6.1 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index cd4f3a7dfa3008a89a74b0fb45c909e7b0b4472f..be8497335f771216b414207d1d241668b9de187b 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.5.2 + 4.6.1 4.0.0 diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index 92371f1e2c3350ae42ec06b46a6d035be6241b73..1ca3fe4c2f482a286b485021544d74dca16d39bf 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.TopicConfig; @@ -183,6 +184,11 @@ public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { return defaultMQAdminExtImpl.examineBrokerClusterAclVersionInfo(addr); } + @Override public AclConfig examineBrokerClusterAclConfig( + String addr) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.examineBrokerClusterAclConfig(addr); + } + @Override public void createAndUpdateSubscriptionGroupConfig(String addr, SubscriptionGroupConfig config) throws RemotingException, diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index 210d5a997d4d5d4c7171d71a8e1af91b47b12e99..22d4005ce9a9e031c0b10e1d09ae6b1a471cc65c 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -41,6 +41,7 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.admin.ConsumeStats; import org.apache.rocketmq.common.admin.OffsetWrapper; @@ -116,7 +117,7 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { this.defaultMQAdminExt.changeInstanceNameToPID(); - this.mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQAdminExt, rpcHook); + this.mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQAdminExt, rpcHook); boolean registerOK = mqClientInstance.registerAdminExt(this.defaultMQAdminExt.getAdminExtGroup(), this); if (!registerOK) { @@ -202,6 +203,12 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { return this.mqClientInstance.getMQClientAPIImpl().getBrokerClusterAclInfo(addr, timeoutMillis); } + @Override + public AclConfig examineBrokerClusterAclConfig( + String addr) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return this.mqClientInstance.getMQClientAPIImpl().getBrokerClusterConfig(addr, timeoutMillis); + } + @Override public void createAndUpdateSubscriptionGroupConfig(String addr, SubscriptionGroupConfig config) throws RemotingException, diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java index d5c75f0e82bfb6420cab03edb4253d2db232bef2..17b62251c60d5af461db085bb8587fbcf9f14f4f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.rocketmq.client.MQAdmin; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.admin.ConsumeStats; @@ -82,6 +83,9 @@ public interface MQAdminExt extends MQAdmin { ClusterAclVersionInfo examineBrokerClusterAclVersionInfo(final String addr) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + AclConfig examineBrokerClusterAclConfig(final String addr) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + void createAndUpdateSubscriptionGroupConfig(final String addr, final SubscriptionGroupConfig config) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index 614fed820d086975cb7968650f4cfa85ad8c4b81..28431a963ee676ea5acef4c9ebd0ee8b9e0e2f2e 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -31,6 +31,7 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.srvutil.ServerUtil; import org.apache.rocketmq.tools.command.acl.ClusterAclConfigVersionListSubCommand; +import org.apache.rocketmq.tools.command.acl.GetAccessConfigSubCommand; import org.apache.rocketmq.tools.command.acl.DeleteAccessConfigSubCommand; import org.apache.rocketmq.tools.command.acl.UpdateAccessConfigSubCommand; import org.apache.rocketmq.tools.command.acl.UpdateGlobalWhiteAddrSubCommand; @@ -209,6 +210,7 @@ public class MQAdminStartup { initCommand(new DeleteAccessConfigSubCommand()); initCommand(new ClusterAclConfigVersionListSubCommand()); initCommand(new UpdateGlobalWhiteAddrSubCommand()); + initCommand(new GetAccessConfigSubCommand()); } private static void initLogback() throws JoranException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java new file mode 100644 index 0000000000000000000000000000000000000000..90638c13d3a0e3a33c7ebb1db12ba3cc6b694244 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.acl; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.AclConfig; +import org.apache.rocketmq.common.PlainAccessConfig; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.CommandUtil; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Set; + +public class GetAccessConfigSubCommand implements SubCommand { + @Override public String commandName() { + return "getAccessConfigSubCommand"; + } + + @Override public String commandDesc() { + return "List all of acl config information in cluster"; + } + + @Override public Options buildCommandlineOptions(Options options) { + OptionGroup optionGroup = new OptionGroup(); + + Option opt = new Option("b", "brokerAddr", true, "query acl config version for which broker"); + optionGroup.addOption(opt); + + opt = new Option("c", "clusterName", true, "query acl config version for specified cluster"); + optionGroup.addOption(opt); + + optionGroup.setRequired(true); + options.addOptionGroup(optionGroup); + + return options; + } + + @Override public void execute(CommandLine commandLine, Options options, + RPCHook rpcHook) throws SubCommandException { + + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + defaultMQAdminExt.start(); + printClusterBaseInfo(defaultMQAdminExt, addr); + return; + + } else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + + defaultMQAdminExt.start(); + + Set masterSet = + CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String addr : masterSet) { + printClusterBaseInfo(defaultMQAdminExt, addr); + } + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + private void printClusterBaseInfo( + final DefaultMQAdminExt defaultMQAdminExt, final String addr) throws + InterruptedException, MQBrokerException, RemotingException, MQClientException, IllegalAccessException { + AclConfig aclConfig = defaultMQAdminExt.examineBrokerClusterAclConfig(addr); + List configs = aclConfig.getPlainAccessConfigs(); + List globalWhiteAddrs = aclConfig.getGlobalWhiteAddrs(); + System.out.printf("\n"); + System.out.printf("%-20s: %s\n", "globalWhiteRemoteAddresses", globalWhiteAddrs.toString()); + System.out.printf("\n"); + System.out.printf("accounts:\n"); + if (configs != null && configs.size() > 0) { + for (PlainAccessConfig config : configs) { + Field[] fields = config.getClass().getDeclaredFields(); + for (Field field : fields) { + field.setAccessible(true); + if (field.get(config) != null) { + System.out.printf("%-1s %-18s: %s\n", "", field.getName(), field.get(config).toString()); + } else { + System.out.printf("%-1s %-18s: %s\n", "", field.getName(), ""); + } + } + System.out.printf("\n"); + } + } + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java index ecbe1f38a54def3368ec02e2f1893f516e29f73d..5038123561bf0aafb880ee0100d3e3323c2c49ae 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java @@ -53,7 +53,7 @@ public class CLusterSendMsgRTCommand implements SubCommand { @Override public Options buildCommandlineOptions(Options options) { - Option opt = new Option("a", "amout", true, "message amout | default 100"); + Option opt = new Option("a", "amount", true, "message amount | default 100"); opt.setRequired(false); options.addOption(opt); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java index 14cd720615bceb56e624fc69b901f0cccde4df22..404d90d669212cc610530e87e1a23d0fd06164b0 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java @@ -49,7 +49,7 @@ public class CheckMsgSendRTCommand implements SubCommand { opt.setRequired(true); options.addOption(opt); - opt = new Option("a", "amout", true, "message amout | default 100"); + opt = new Option("a", "amount", true, "message amount | default 100"); opt.setRequired(false); options.addOption(opt); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java index 6bb8caad40ad5e4a89816f7f7c5f5a6003223b77..bdcd5c8a27fef681350c41120ba0237c6459c157 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java @@ -66,11 +66,6 @@ public class QueryMsgByIdSubCommand implements SubCommand { msgId ); - System.out.printf("%-20s %s%n", - "OffsetID:", - msgId - ); - System.out.printf("%-20s %s%n", "Topic:", msg.getTopic() diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java index 69420913e46c9b324c2fcb6161570974cf234e1f..a06a19d1fd08e800d51ffe58bae006b3140ee8df 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java @@ -16,12 +16,16 @@ */ package org.apache.rocketmq.tools.command.topic; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.remoting.RPCHook; @@ -67,46 +71,89 @@ public class UpdateTopicPermSubCommand implements SubCommand { @Override public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) throws SubCommandException { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { defaultMQAdminExt.start(); - TopicConfig topicConfig = new TopicConfig(); - String topic = commandLine.getOptionValue('t').trim(); + TopicConfig topicConfig = new TopicConfig(); + String topic; + if (commandLine.hasOption('t')) { + topic = commandLine.getOptionValue('t').trim(); + } else { + System.out.printf("topic parameter value must be need.%n"); + return; + } TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); assert topicRouteData != null; List queueDatas = topicRouteData.getQueueDatas(); assert queueDatas != null && queueDatas.size() > 0; - QueueData queueData = queueDatas.get(0); topicConfig.setTopicName(topic); topicConfig.setWriteQueueNums(queueData.getWriteQueueNums()); topicConfig.setReadQueueNums(queueData.getReadQueueNums()); - topicConfig.setPerm(queueData.getPerm()); topicConfig.setTopicSysFlag(queueData.getTopicSynFlag()); - //new perm - int perm = Integer.parseInt(commandLine.getOptionValue('p').trim()); - int oldPerm = topicConfig.getPerm(); - if (perm == oldPerm) { - System.out.printf("new perm equals to the old one!%n"); + int perm; + if (commandLine.hasOption('p')) { + perm = Integer.parseInt(commandLine.getOptionValue('p').trim()); + } else { + System.out.printf("perm parameter value must be need.%n"); return; } topicConfig.setPerm(perm); if (commandLine.hasOption('b')) { - String addr = commandLine.getOptionValue('b').trim(); - defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); - System.out.printf("update topic perm from %s to %s in %s success.%n", oldPerm, perm, addr); - System.out.printf("%s%n", topicConfig); - return; + String brokerAddr = commandLine.getOptionValue('b').trim(); + List brokerDatas = topicRouteData.getBrokerDatas(); + String brokerName = null; + for (BrokerData data : brokerDatas) { + HashMap brokerAddrs = data.getBrokerAddrs(); + if (brokerAddrs == null || brokerAddrs.size() == 0) { + continue; + } + for (Map.Entry entry : brokerAddrs.entrySet()) { + if (brokerAddr.equals(entry.getValue()) && MixAll.MASTER_ID == entry.getKey()) { + brokerName = data.getBrokerName(); + break; + } + } + if (brokerName != null) { + break; + } + } + + if (brokerName != null) { + List queueDataList = topicRouteData.getQueueDatas(); + assert queueDataList != null && queueDataList.size() > 0; + int oldPerm = 0; + for (QueueData data : queueDataList) { + if (brokerName.equals(data.getBrokerName())) { + oldPerm = data.getPerm(); + if (perm == oldPerm) { + System.out.printf("new perm equals to the old one!%n"); + return; + } + break; + } + } + defaultMQAdminExt.createAndUpdateTopicConfig(brokerAddr, topicConfig); + System.out.printf("update topic perm from %s to %s in %s success.%n", oldPerm, perm, brokerAddr); + System.out.printf("%s.%n", topicConfig); + return; + } else { + System.out.printf("updateTopicPerm error broker not exit or broker is not master!.%n"); + return; + } + } else if (commandLine.hasOption('c')) { String clusterName = commandLine.getOptionValue('c').trim(); Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); for (String addr : masterSet) { defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); - System.out.printf("update topic perm from %s to %s in %s success.%n", oldPerm, perm, addr); + System.out.printf("update topic perm from %s to %s in %s success.%n", queueData.getPerm(), perm, addr); } return; } diff --git a/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java b/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java index 78659800219742a7b2cf19cf188679d2584b13b8..3146b1781154792a2d7928e0306e9cf172b5ddac 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java @@ -85,7 +85,7 @@ import static org.mockito.Mockito.when; public class DefaultMQAdminExtTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; private static Properties properties = new Properties(); private static TopicList topicList = new TopicList(); diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/CommandUtilTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/CommandUtilTest.java index 33b449768d105b6d223c84d6b46ae5d22beba916..b556e5c1975bf5f2ccee6cccfa9d0a3548c9060b 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/CommandUtilTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/CommandUtilTest.java @@ -48,7 +48,7 @@ import static org.mockito.Mockito.when; public class CommandUtilTest { private DefaultMQAdminExt defaultMQAdminExt; private DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private MQClientAPIImpl mQClientAPIImpl; @Before diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommandTest.java new file mode 100644 index 0000000000000000000000000000000000000000..6a7694ef0a2a83bfb62e2f98f51d3f3f1e6df5cb --- /dev/null +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommandTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.acl; + +import org.apache.commons.cli.*; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class GetAccessConfigSubCommandTest { + + @Test + public void testExecute() { + GetAccessConfigSubCommand cmd = new GetAccessConfigSubCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = new String[] {"-c default-cluster"}; + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, cmd.buildCommandlineOptions(options), new PosixParser()); + assertThat(commandLine.getOptionValue('c').trim()).isEqualTo("default-cluster"); + } +} diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommadTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommadTest.java index 1089c1b70cd7475503662c9a8c71a78a7e7dfd6c..1abd8575b0d2817f045fe3bb563e05a8c36c5a24 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommadTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommadTest.java @@ -39,7 +39,6 @@ import org.apache.rocketmq.tools.admin.DefaultMQAdminExtImpl; import org.apache.rocketmq.tools.command.SubCommandException; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -54,7 +53,7 @@ public class BrokerConsumeStatsSubCommadTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommandTest.java index b1d92d79a3fc8ae24f7c3912cc8bbf7ad5e75f24..c850d71d881534d4006645c494fd4731814b47e4 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommandTest.java @@ -48,7 +48,7 @@ import static org.mockito.Mockito.when; public class BrokerStatusSubCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/CleanExpiredCQSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/CleanExpiredCQSubCommandTest.java index a5ba24f6503187a81a0230c510e9e37ae9dccd42..241ae8829abfb28349e9d2fc537bf29dc4c6fe8b 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/CleanExpiredCQSubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/CleanExpiredCQSubCommandTest.java @@ -46,7 +46,7 @@ import static org.mockito.Mockito.when; public class CleanExpiredCQSubCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/CleanUnusedTopicCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/CleanUnusedTopicCommandTest.java index 95373800727c5a3f3209ecfb0978c66da41a2fcf..759f783e49522b9ac5a721986fd2d66e506a936c 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/CleanUnusedTopicCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/CleanUnusedTopicCommandTest.java @@ -46,7 +46,7 @@ import static org.mockito.Mockito.when; public class CleanUnusedTopicCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommandTest.java index 978a2fd59616c19347c3d6186f7ea58fcf9f519a..8bb40793790312e7a3bc36feb1dd4a4a11175fe6 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommandTest.java @@ -48,7 +48,7 @@ import static org.mockito.Mockito.when; public class GetBrokerConfigCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/SendMsgStatusCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/SendMsgStatusCommandTest.java index c0f7639d93dfdcc1a5e2a6ccb96a30b399956d47..9e9bc789d93eb0c95169bf44b01bdaaac40cfddc 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/SendMsgStatusCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/SendMsgStatusCommandTest.java @@ -41,7 +41,7 @@ import static org.mockito.Mockito.mock; public class SendMsgStatusCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommandTest.java index 46c6eb3e74596f0d48a2469b7acd8de404a6e6b5..c74107edf921be37b0af4b818a3e0b307d74a2bd 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommandTest.java @@ -44,7 +44,7 @@ import static org.mockito.Mockito.mock; public class UpdateBrokerConfigSubCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommandTest.java index 3d64ce2422cee9f6342576cbaad91fb397432dbe..584943ce442ad29d105a2dd9a682cec2b07ddeed 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommandTest.java @@ -54,7 +54,7 @@ import static org.mockito.Mockito.when; public class ConsumerConnectionSubCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommandTest.java index 76c8dc4b3ccc50a5363e59e15fabd7640332f240..060ba9383d5fc4fc3caef2f2d08275b0cf952d0e 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommandTest.java @@ -49,7 +49,7 @@ import static org.mockito.Mockito.when; public class ProducerConnectionSubCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommandTest.java index 6d69c10b5abbe752f2cfedb43177e6ca3de226d3..19d903cea6f8151313e68e566f977d7d65d66315 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommandTest.java @@ -53,7 +53,7 @@ import static org.mockito.Mockito.when; public class ConsumerProgressSubCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommandTest.java index aa6f27de09f0881486338025453e63d72b541a52..7f44af8225ce6968623d639c7ad9f983ee96502b 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommandTest.java @@ -65,7 +65,7 @@ import static org.mockito.Mockito.when; public class ConsumerStatusSubCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java index e757608280ff8f912f95563e72a6734422712060..504b46567808c98bb77de104e13b5c5f37aeb9c4 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java @@ -61,7 +61,7 @@ public class QueryMsgByUniqueKeySubCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; private static MQAdminImpl mQAdminImpl; diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/GetNamesrvConfigCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/GetNamesrvConfigCommandTest.java index 8163fd15bc939c1aa1f7a978df5d59d048ad5b60..dde80eb3e672165338cc6a50928bc19255c2975c 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/GetNamesrvConfigCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/GetNamesrvConfigCommandTest.java @@ -41,7 +41,6 @@ import org.apache.rocketmq.tools.admin.DefaultMQAdminExtImpl; import org.apache.rocketmq.tools.command.SubCommandException; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.mockito.ArgumentMatchers; @@ -52,7 +51,7 @@ import static org.mockito.Mockito.when; public class GetNamesrvConfigCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/UpdateKvConfigCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/UpdateKvConfigCommandTest.java index af4deace0a9cd563944a336e8548f97f5d992797..c4edcafe6f2dc52c49a840b059f742989401b145 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/UpdateKvConfigCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/UpdateKvConfigCommandTest.java @@ -39,7 +39,7 @@ import static org.mockito.Mockito.mock; public class UpdateKvConfigCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommandTest.java index 11711d08866e80f2f185fbfc6cef15b325f0203e..9befdf8948ab016ee920c43d9f307eae3de26818 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommandTest.java @@ -49,7 +49,7 @@ import static org.mockito.Mockito.when; public class WipeWritePermSubCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/offset/GetConsumerStatusCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/offset/GetConsumerStatusCommandTest.java index 86454008c220cdcc5f4d88bc78a5cabfc5971a40..a01bf8167ae65c36046a97fa495686020a691002 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/offset/GetConsumerStatusCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/offset/GetConsumerStatusCommandTest.java @@ -47,7 +47,7 @@ import static org.mockito.Mockito.when; public class GetConsumerStatusCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommandTest.java index b7af8c870b56d62a58738d16ee687d5cdebc1687..d73a996b3fbcd60a73f998cf2475340b5562bbb3 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommandTest.java @@ -53,7 +53,7 @@ import static org.mockito.Mockito.when; public class ResetOffsetByTimeCommandTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; @BeforeClass diff --git a/tools/src/test/java/org/apache/rocketmq/tools/monitor/MonitorServiceTest.java b/tools/src/test/java/org/apache/rocketmq/tools/monitor/MonitorServiceTest.java index 4989a9b55ab37920a97415fbae24de81a7334300..57278b9b222fd61b0227c87ab4d2445f8da73408 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/monitor/MonitorServiceTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/monitor/MonitorServiceTest.java @@ -69,7 +69,7 @@ import static org.mockito.Mockito.when; public class MonitorServiceTest { private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; - private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private static MQClientAPIImpl mQClientAPIImpl; private static MonitorConfig monitorConfig; private static MonitorListener monitorListener;