diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 143dac89a96f92a03f173b5bf7c64715b59ccf52..1eb8c8343821d95aae1a2d2e12f33016bec752e0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,6 @@ + +**Make sure set the target branch to `develop`** + ## What is the purpose of the change XXXXX diff --git a/.travis.yml b/.travis.yml index 34d38c2c54d5e87ee55ab2a10afa848444be3f30..df6a7356bffa1e505dd70b950bd6af0954d1d5ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: bionic notifications: email: @@ -9,22 +9,38 @@ notifications: language: java -jdk: - - oraclejdk8 - matrix: include: # On OSX, run with default JDK only. # - os: osx - # On Linux, run with specific JDKs only. - - os: linux - env: CUSTOM_JDK="oraclejdk8" + # On Linux we install latest OpenJDK 1.8 from Ubuntu repositories + - name: Linux x86_64 + arch: amd64 + - name: Linux aarch64 + arch: arm64 + +cache: + directories: + - $HOME/.m2/repository before_install: + - lscpu - echo 'MAVEN_OPTS="$MAVEN_OPTS -Xmx1024m -XX:MaxPermSize=512m -XX:+BytecodeVerificationLocal"' >> ~/.mavenrc - cat ~/.mavenrc - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export JAVA_HOME=$(/usr/libexec/java_home); fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then jdk_switcher use "$CUSTOM_JDK"; fi + +install: | + if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + sudo apt update + sudo apt install -y openjdk-8-jdk maven + export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-${TRAVIS_CPU_ARCH}/" + export PATH="$JAVA_HOME/bin:/usr/share/maven/bin:$PATH" + fi + +before_script: + - java -version + - mvn -version + - ulimit -c unlimited script: - travis_retry mvn -B clean apache-rat:check diff --git a/LICENSE b/LICENSE index 7f77f44e739b555b76538b83fe898a4d786c554d..f49a4e16e68b128803cc2dcea614603632b04eac 100644 --- a/LICENSE +++ b/LICENSE @@ -15,7 +15,7 @@ "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, - "control" means (properties) the power, direct or indirect, to cause the + "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. diff --git a/NOTICE b/NOTICE index 65ebdd036106a171f325936cdc0a907b0d6400c5..086ee9fa175f778d120c5dd4a7b81600595b51cf 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache RocketMQ -Copyright 2016-2020 The Apache Software Foundation +Copyright 2016-2021 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/README.md b/README.md index 9b28078f936269af1708ac86e84da8b71059c0fb..1b1398cb792644acc98420072c1ce844e5cac275 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/apache/rocketmq.svg)](http://isitmaintained.com/project/apache/rocketmq "Average time to resolve an issue") [![Percentage of issues still open](http://isitmaintained.com/badge/open/apache/rocketmq.svg)](http://isitmaintained.com/project/apache/rocketmq "Percentage of issues still open") -![Twitter Follow](https://img.shields.io/twitter/follow/ApacheRocketMQ?style=social) +[![Twitter Follow](https://img.shields.io/twitter/follow/ApacheRocketMQ?style=social)](https://twitter.com/intent/follow?screen_name=ApacheRocketMQ) **[Apache RocketMQ](https://rocketmq.apache.org) is a distributed messaging and streaming platform with low latency, high performance and reliability, trillion-level capacity and flexible scalability.** It offers a variety of features: -* Messageing patterns including publish/subscribe, request/reply and streaming +* Messaging patterns including publish/subscribe, request/reply and streaming * Financial grade transactional message * Built-in fault tolerance and high availability configuration options base on [DLedger](https://github.com/openmessaging/openmessaging-storage-dledger) * A variety of cross language clients, such as Java, C/C++, Python, Go diff --git a/acl/pom.xml b/acl/pom.xml index bb91ac898823955c5235df6c06b11798bc5b3262..d62493417892e67b78a46f414f95320be263b6a7 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,15 +13,11 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT rocketmq-acl rocketmq-acl ${project.version} - http://maven.apache.org - - UTF-8 - ${project.groupId} @@ -62,11 +58,6 @@ logback-classic test - - ch.qos.logback - logback-core - test - commons-validator commons-validator diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java index bfe96f53037cca259b837a461114f879b2b7359c..d129c66d1c81dbb0cbce7dc7d5324eacdd53864d 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java @@ -44,6 +44,16 @@ public class AclConstants { public static final String CONFIG_TIME_STAMP = "timestamp"; + public static final String PUB = "PUB"; + + public static final String SUB = "SUB"; + + public static final String DENY = "DENY"; + + public static final String PUB_SUB = "PUB|SUB"; + + public static final String SUB_PUB = "SUB|PUB"; + public static final int ACCESS_KEY_MIN_LENGTH = 6; public static final int SECRET_KEY_MIN_LENGTH = 6; diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java index 8973320237c684c68864c29ff0c4a90d75bfdd3e..77abe0e80fbe734576e1b557a889f0e0a2afae47 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java @@ -94,7 +94,7 @@ public class AclUtils { } } - public static String v6ipProcess(String netaddress, String[] strArray, int index) { + public static String v6ipProcess(String netaddress) { int part; String subAddress; boolean isAsterisk = isAsterisk(netaddress); @@ -120,7 +120,7 @@ public class AclUtils { } } - public static String[] getAddreeStrArray(String netaddress, String partialAddress) { + public static String[] getAddresses(String netaddress, String partialAddress) { String[] parAddStrArray = StringUtils.split(partialAddress.substring(1, partialAddress.length() - 1), ","); String address = netaddress.substring(0, netaddress.indexOf("{")); String[] addreeStrArray = new String[parAddStrArray.length]; diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java b/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java index 0acc8e9508138f3db3ae508dd5f78cd4a10011a2..dadcaa304aac45a460feb3e9a15654ffa46d6e8b 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java @@ -60,15 +60,14 @@ public class Permission { return Permission.DENY; } switch (permString.trim()) { - case "PUB": + case AclConstants.PUB: return Permission.PUB; - case "SUB": + case AclConstants.SUB: return Permission.SUB; - case "PUB|SUB": + case AclConstants.PUB_SUB: + case AclConstants.SUB_PUB: return Permission.PUB | Permission.SUB; - case "SUB|PUB": - return Permission.PUB | Permission.SUB; - case "DENY": + case AclConstants.DENY: return Permission.DENY; default: return Permission.DENY; @@ -90,6 +89,25 @@ public class Permission { } } + public static void checkResourcePerms(List resources) { + if (resources == null || resources.isEmpty()) { + return; + } + + for (String resource : resources) { + String[] items = StringUtils.split(resource, "="); + if (items.length != 2) { + throw new AclException(String.format("Parse Resource format error for %s.\n" + + "The expected resource format is 'Res=Perm'. For example: topicA=SUB", resource)); + } + + if (!AclConstants.DENY.equals(items[1].trim()) && Permission.DENY == Permission.parsePermFromString(items[1].trim())) { + throw new AclException(String.format("Parse resource permission error for %s.\n" + + "The expected permissions are 'SUB' or 'PUB' or 'SUB|PUB' or 'PUB|SUB'.", resource)); + } + } + } + public static boolean needAdminPerm(Integer code) { return ADMIN_CODE.contains(code); } diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java index 89638f6ac2cdcc818537887fb77851cdb16f0b1e..078d34b10b02a84837ad6bb0b34dd884833cab9c 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java @@ -50,9 +50,9 @@ public class PlainPermissionManager { private String fileName = System.getProperty("rocketmq.acl.plain.file", DEFAULT_PLAIN_ACL_FILE); - private Map plainAccessResourceMap = new HashMap<>(); + private Map plainAccessResourceMap = new HashMap<>(); - private List globalWhiteRemoteAddressStrategy = new ArrayList<>(); + private List globalWhiteRemoteAddressStrategy = new ArrayList<>(); private RemoteAddressStrategyFactory remoteAddressStrategyFactory = new RemoteAddressStrategyFactory(); @@ -73,14 +73,14 @@ public class PlainPermissionManager { JSONObject plainAclConfData = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, JSONObject.class); if (plainAclConfData == null || plainAclConfData.isEmpty()) { - throw new AclException(String.format("%s file is not data", fileHome + File.separator + fileName)); + throw new AclException(String.format("%s file is not data", fileHome + File.separator + fileName)); } log.info("Broker plain acl conf data is : ", plainAclConfData.toString()); JSONArray globalWhiteRemoteAddressesList = plainAclConfData.getJSONArray("globalWhiteRemoteAddresses"); if (globalWhiteRemoteAddressesList != null && !globalWhiteRemoteAddressesList.isEmpty()) { for (int i = 0; i < globalWhiteRemoteAddressesList.size(); i++) { globalWhiteRemoteAddressStrategy.add(remoteAddressStrategyFactory. - getRemoteAddressStrategy(globalWhiteRemoteAddressesList.getString(i))); + getRemoteAddressStrategy(globalWhiteRemoteAddressesList.getString(i))); } } @@ -89,7 +89,7 @@ public class PlainPermissionManager { List plainAccessConfigList = accounts.toJavaList(PlainAccessConfig.class); for (PlainAccessConfig plainAccessConfig : plainAccessConfigList) { PlainAccessResource plainAccessResource = buildPlainAccessResource(plainAccessConfig); - plainAccessResourceMap.put(plainAccessResource.getAccessKey(),plainAccessResource); + plainAccessResourceMap.put(plainAccessResource.getAccessKey(), plainAccessResource); } } @@ -128,12 +128,17 @@ public class PlainPermissionManager { if (plainAccessConfig == null) { log.error("Parameter value plainAccessConfig is null,Please check your parameter"); - return false; + throw new AclException("Parameter value plainAccessConfig is null, Please check your parameter"); } + Permission.checkResourcePerms(plainAccessConfig.getTopicPerms()); + Permission.checkResourcePerms(plainAccessConfig.getGroupPerms()); + Map aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, Map.class); - + if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) { + throw new AclException(String.format("the %s file is not found or empty", fileHome + File.separator + fileName)); + } List> accounts = (List>) aclAccessConfigMap.get(AclConstants.CONFIG_ACCOUNTS); Map updateAccountMap = null; if (accounts != null) { @@ -164,21 +169,21 @@ public class PlainPermissionManager { return false; } - private Map createAclAccessConfigMap(Map existedAccoutMap, PlainAccessConfig plainAccessConfig) { - + private Map createAclAccessConfigMap(Map existedAccountMap, + PlainAccessConfig plainAccessConfig) { Map newAccountsMap = null; - if (existedAccoutMap == null) { + if (existedAccountMap == null) { newAccountsMap = new LinkedHashMap(); } else { - newAccountsMap = existedAccoutMap; + newAccountsMap = existedAccountMap; } if (StringUtils.isEmpty(plainAccessConfig.getAccessKey()) || plainAccessConfig.getAccessKey().length() <= AclConstants.ACCESS_KEY_MIN_LENGTH) { throw new AclException(String.format( - "The accessKey=%s cannot be null and length should longer than 6", - plainAccessConfig.getAccessKey())); + "The accessKey=%s cannot be null and length should longer than 6", + plainAccessConfig.getAccessKey())); } newAccountsMap.put(AclConstants.CONFIG_ACCESS_KEY, plainAccessConfig.getAccessKey()); @@ -219,8 +224,10 @@ public class PlainPermissionManager { } Map aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, - Map.class); - + Map.class); + if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) { + throw new AclException(String.format("the %s file is not found or empty", fileHome + File.separator + fileName)); + } List> accounts = (List>) aclAccessConfigMap.get("accounts"); if (accounts != null) { Iterator> itemIterator = accounts.iterator(); @@ -252,7 +259,9 @@ public class PlainPermissionManager { Map aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, Map.class); - + if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) { + throw new AclException(String.format("the %s file is not found or empty", fileHome + File.separator + fileName)); + } List globalWhiteRemoteAddrList = (List) aclAccessConfigMap.get(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS); if (globalWhiteRemoteAddrList != null) { @@ -260,7 +269,7 @@ public class PlainPermissionManager { globalWhiteRemoteAddrList.addAll(globalWhiteAddrsList); // Update globalWhiteRemoteAddr element in memeory map firstly - aclAccessConfigMap.put(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS,globalWhiteRemoteAddrList); + aclAccessConfigMap.put(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS, globalWhiteRemoteAddrList); if (AclUtils.writeDataObject(fileHome + File.separator + fileName, updateAclConfigFileVersion(aclAccessConfigMap))) { return true; } @@ -276,9 +285,9 @@ public class PlainPermissionManager { List configs = new ArrayList<>(); List whiteAddrs = new ArrayList<>(); JSONObject plainAclConfData = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, - JSONObject.class); + JSONObject.class); if (plainAclConfData == null || plainAclConfData.isEmpty()) { - throw new AclException(String.format("%s file is not data", fileHome + File.separator + fileName)); + throw new AclException(String.format("%s file is not data", fileHome + File.separator + fileName)); } JSONArray globalWhiteAddrs = plainAclConfData.getJSONArray(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS); if (globalWhiteAddrs != null && !globalWhiteAddrs.isEmpty()) { @@ -360,7 +369,7 @@ public class PlainPermissionManager { || plainAccessConfig.getSecretKey().length() <= AclConstants.SECRET_KEY_MIN_LENGTH) { throw new AclException(String.format( "The accessKey=%s and secretKey=%s cannot be null and length should longer than 6", - plainAccessConfig.getAccessKey(), plainAccessConfig.getSecretKey())); + plainAccessConfig.getAccessKey(), plainAccessConfig.getSecretKey())); } PlainAccessResource plainAccessResource = new PlainAccessResource(); plainAccessResource.setAccessKey(plainAccessConfig.getAccessKey()); @@ -376,7 +385,7 @@ public class PlainPermissionManager { Permission.parseResourcePerms(plainAccessResource, true, plainAccessConfig.getTopicPerms()); plainAccessResource.setRemoteAddressStrategy(remoteAddressStrategyFactory. - getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress())); + getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress())); return plainAccessResource; } diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java index 6931eb7c426aa3511769f5e9fa8a4abc6c260bb3..6ec90ee3fad619f36517f1f96b5c40cf020d324b 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java @@ -52,14 +52,18 @@ public class RemoteAddressStrategyFactory { if (!last.startsWith("{")) { throw new AclException(String.format("MultipleRemoteAddressStrategy netaddress examine scope Exception netaddress", remoteAddr)); } - return new MultipleRemoteAddressStrategy(AclUtils.getAddreeStrArray(remoteAddr, last)); + return new MultipleRemoteAddressStrategy(AclUtils.getAddresses(remoteAddr, last)); } else { String[] strArray = StringUtils.split(remoteAddr, "."); - String four = strArray[3]; - if (!four.startsWith("{")) { + // However a right IP String provided by user,it always can be divided into 4 parts by '.'. + if (strArray.length < 4) { + throw new AclException(String.format("MultipleRemoteAddressStrategy has got a/some wrong format IP(s) ", remoteAddr)); + } + String lastStr = strArray[strArray.length - 1]; + if (!lastStr.startsWith("{")) { throw new AclException(String.format("MultipleRemoteAddressStrategy netaddress examine scope Exception netaddress", remoteAddr)); } - return new MultipleRemoteAddressStrategy(AclUtils.getAddreeStrArray(remoteAddr, four)); + return new MultipleRemoteAddressStrategy(AclUtils.getAddresses(remoteAddr, lastStr)); } } else if (AclUtils.isComma(remoteAddr)) { return new MultipleRemoteAddressStrategy(StringUtils.split(remoteAddr, ",")); @@ -153,7 +157,7 @@ public class RemoteAddressStrategyFactory { for (int i = 1; i < strArray.length; i++) { if (ipv6Analysis(strArray, i)) { AclUtils.verify(remoteAddr, index - 1); - String preAddress = AclUtils.v6ipProcess(remoteAddr, strArray, index); + String preAddress = AclUtils.v6ipProcess(remoteAddr); this.index = StringUtils.split(preAddress, ":").length; this.head = preAddress; break; @@ -189,7 +193,7 @@ public class RemoteAddressStrategyFactory { throw new AclException(String.format("RangeRemoteAddressStrategy netaddress examine scope Exception start is %s , end is %s", start, end)); } } - return this.end > 0 ? true : false; + return this.end > 0; } private boolean ipv6Analysis(String[] strArray, int index) { diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java index 5705b745a0e7472ca7a215468ae20775a53babfb..7df0afa6393346eccfc4f8bc48e1952d8f746ede 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java @@ -32,9 +32,9 @@ import org.junit.Test; public class AclUtilsTest { @Test - public void getAddreeStrArray() { + public void getAddresses() { String address = "1.1.1.{1,2,3,4}"; - String[] addressArray = AclUtils.getAddreeStrArray(address, "{1,2,3,4}"); + String[] addressArray = AclUtils.getAddresses(address, "{1,2,3,4}"); List newAddressList = new ArrayList<>(); for (String a : addressArray) { newAddressList.add(a); @@ -49,7 +49,7 @@ public class AclUtilsTest { // IPv6 test String ipv6Address = "1:ac41:9987::bb22:666:{1,2,3,4}"; - String[] ipv6AddressArray = AclUtils.getAddreeStrArray(ipv6Address, "{1,2,3,4}"); + String[] ipv6AddressArray = AclUtils.getAddresses(ipv6Address, "{1,2,3,4}"); List newIPv6AddressList = new ArrayList<>(); for (String a : ipv6AddressArray) { newIPv6AddressList.add(a); @@ -181,19 +181,23 @@ public class AclUtilsTest { public void v6ipProcessTest() { String remoteAddr = "5::7:6:1-200:*"; String[] strArray = StringUtils.split(remoteAddr, ":"); - Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0000:0000:0000:0007:0006"); + Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr), "0005:0000:0000:0000:0007:0006"); +// Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0000:0000:0000:0007:0006"); remoteAddr = "5::7:6:1-200"; strArray = StringUtils.split(remoteAddr, ":"); - Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0000:0000:0000:0000:0007:0006"); + Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr), "0005:0000:0000:0000:0000:0007:0006"); +// Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0000:0000:0000:0000:0007:0006"); remoteAddr = "5::7:6:*"; strArray = StringUtils.split(remoteAddr, ":"); - Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0000:0000:0000:0000:0007:0006"); + Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr), "0005:0000:0000:0000:0000:0007:0006"); +// Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0000:0000:0000:0000:0007:0006"); remoteAddr = "5:7:6:*"; strArray = StringUtils.split(remoteAddr, ":"); - Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0007:0006"); + Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr), "0005:0007:0006"); +// Assert.assertEquals(AclUtils.v6ipProcess(remoteAddr, strArray, 3), "0005:0007:0006"); } @Test diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java index 253b5b241e05a9e900f5c730e0cec8094fbe2d98..c824065f7ff80241df142321642d9419c938c436 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.acl.common; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -165,4 +166,27 @@ public class PermissionTest { aclException.setStatus("netaddress examine scope Exception netaddress"); Assert.assertEquals(aclException.getStatus(),"netaddress examine scope Exception netaddress"); } + + @Test + public void checkResourcePermsNormalTest() { + Permission.checkResourcePerms(null); + Permission.checkResourcePerms(new ArrayList<>()); + Permission.checkResourcePerms(Arrays.asList("topicA=PUB")); + Permission.checkResourcePerms(Arrays.asList("topicA=PUB", "topicB=SUB", "topicC=PUB|SUB")); + } + + @Test(expected = AclException.class) + public void checkResourcePermsExceptionTest1() { + Permission.checkResourcePerms(Arrays.asList("topicA")); + } + + @Test(expected = AclException.class) + public void checkResourcePermsExceptionTest2() { + Permission.checkResourcePerms(Arrays.asList("topicA=")); + } + + @Test(expected = AclException.class) + public void checkResourcePermsExceptionTest3() { + Permission.checkResourcePerms(Arrays.asList("topicA=DENY1")); + } } diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java index 645e5227631dac402dc11ca9a44fa2a2fdd8b380..c3d41919a7e824efa93d911a7bd4ca81390f787b 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java @@ -546,6 +546,26 @@ public class PlainAccessValidatorTest { Assert.assertEquals(plainAccessValidator.updateAccessConfig(plainAccessConfig), false); } + @Test(expected = AclException.class) + public void createAndUpdateAccessAclYamlConfigExceptionTest() { + System.setProperty("rocketmq.home.dir", "src/test/resources"); + System.setProperty("rocketmq.acl.plain.file", "/conf/plain_acl_update_create.yml"); + + PlainAccessConfig plainAccessConfig = new PlainAccessConfig(); + plainAccessConfig.setAccessKey("RocketMQ33"); + plainAccessConfig.setSecretKey("123456789111"); + List topicPerms = new ArrayList(); + topicPerms.add("topicB=PUB"); + plainAccessConfig.setTopicPerms(topicPerms); + List groupPerms = new ArrayList(); + groupPerms.add("groupC=DENY1"); + plainAccessConfig.setGroupPerms(groupPerms); + + PlainAccessValidator plainAccessValidator = new PlainAccessValidator(); + // Create element in the acl access yaml config file + plainAccessValidator.updateAccessConfig(plainAccessConfig); + } + @Test public void updateGlobalWhiteAddrsNormalTest() { System.setProperty("rocketmq.home.dir", "src/test/resources"); diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java index 8998dd993ccbcf75c9788fd8cf314c4c93f58771..87eb37bdf2f18e73f3c099523118b81c30f957dd 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyTest.java @@ -198,6 +198,16 @@ public class RemoteAddressStrategyTest { remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); plainAccessResource.setWhiteRemoteAddress("::1,2,3}"); remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress("192.168.1.{1}"); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress("192.168.1.{1,2}"); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress("192.168.{1}"); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress("{192.168.1}"); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); + plainAccessResource.setWhiteRemoteAddress("{192.168.1.1}"); + remoteAddressStrategyFactory.getRemoteAddressStrategy(plainAccessResource); } private void multipleNetaddressStrategyTest(RemoteAddressStrategy remoteAddressStrategy) { diff --git a/broker/pom.xml b/broker/pom.xml index 275758f928b1682ad953139caad5fbfda179e777..3a58aa0c717d542afa684680daa640f662fe16b4 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -54,10 +54,6 @@ ch.qos.logback logback-classic - - ch.qos.logback - logback-core - com.alibaba fastjson diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 860b3493d1e97c93b178b48e9717aaf328510552..4bd00eff181e51655279166d026b58152e2c0d94 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -33,7 +33,7 @@ import org.apache.rocketmq.remoting.common.RemotingUtil; public class ProducerManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; - private static final int GET_AVALIABLE_CHANNEL_RETRY_COUNT = 3; + private static final int GET_AVAILABLE_CHANNEL_RETRY_COUNT = 3; private final ConcurrentHashMap> groupChannelTable = new ConcurrentHashMap<>(); private final ConcurrentHashMap clientChannelTable = new ConcurrentHashMap<>(); @@ -131,16 +131,14 @@ public class ProducerManager { } } - public Channel getAvaliableChannel(String groupId) { + public Channel getAvailableChannel(String groupId) { if (groupId == null) { return null; } - List channelList = new ArrayList(); + List channelList; ConcurrentHashMap channelClientChannelInfoHashMap = groupChannelTable.get(groupId); if (channelClientChannelInfoHashMap != null) { - for (Channel channel : channelClientChannelInfoHashMap.keySet()) { - channelList.add(channel); - } + channelList = new ArrayList<>(channelClientChannelInfoHashMap.keySet()); } else { log.warn("Check transaction failed, channel table is empty. groupId={}", groupId); return null; @@ -158,7 +156,7 @@ public class ProducerManager { Channel channel = channelList.get(index); int count = 0; boolean isOk = channel.isActive() && channel.isWritable(); - while (count++ < GET_AVALIABLE_CHANNEL_RETRY_COUNT) { + while (count++ < GET_AVAILABLE_CHANNEL_RETRY_COUNT) { if (isOk) { return channel; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java b/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java index a74b6d66ce8517c08c608744ce489575da1e12cc..a89bace193e6fb64c60796172998a835bde4333e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java @@ -17,9 +17,9 @@ package org.apache.rocketmq.broker.mqtrace; public interface SendMessageHook { - public String hookName(); + String hookName(); - public void sendMessageBefore(final SendMessageContext context); + void sendMessageBefore(final SendMessageContext context); - public void sendMessageAfter(final SendMessageContext context); + void sendMessageAfter(final SendMessageContext context); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java index ebc9dd8acce90512f1f0ddc17c62bbd866057729..bd0575875970f29fd4dc97d14bfcedbb762c2cab 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java @@ -232,4 +232,20 @@ public class ConsumerOffsetManager extends ConfigManager { } } + public void removeOffset(final String group) { + Iterator>> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topicAtGroup = next.getKey(); + if (topicAtGroup.contains(group)) { + String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR); + if (arrays.length == 2 && group.equals(arrays[1])) { + it.remove(); + log.warn("clean group offset {}", topicAtGroup); + } + } + } + + } + } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index 6caa2358e32eb9ece9be1e14f3fc158b3406d034..de7f3fce81c7a70a7e9b0eeb1daa6a2f9dd68c4c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.broker.out; -import com.google.common.collect.Lists; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.List; @@ -123,7 +122,7 @@ public class BrokerOuterAPI { final int timeoutMills, final boolean compressed) { - final List registerBrokerResultList = Lists.newArrayList(); + final List registerBrokerResultList = new CopyOnWriteArrayList<>(); List nameServerAddressList = this.remotingClient.getNameServerAddressList(); if (nameServerAddressList != null && nameServerAddressList.size() > 0) { @@ -147,7 +146,7 @@ public class BrokerOuterAPI { @Override public void run() { try { - RegisterBrokerResult result = registerBroker(namesrvAddr,oneway, timeoutMills,requestHeader,body); + RegisterBrokerResult result = registerBroker(namesrvAddr, oneway, timeoutMills, requestHeader, body); if (result != null) { registerBrokerResultList.add(result); } @@ -209,7 +208,7 @@ public class BrokerOuterAPI { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), requestHeader == null ? null : requestHeader.getBrokerAddr()); } public void unregisterBrokerAll( @@ -255,7 +254,7 @@ public class BrokerOuterAPI { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr); } public List needRegister( @@ -338,7 +337,7 @@ public class BrokerOuterAPI { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public ConsumerOffsetSerializeWrapper getAllConsumerOffset( @@ -355,7 +354,7 @@ public class BrokerOuterAPI { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public String getAllDelayOffset( @@ -372,7 +371,7 @@ public class BrokerOuterAPI { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public SubscriptionGroupWrapper getAllSubscriptionGroupConfig( @@ -389,7 +388,7 @@ public class BrokerOuterAPI { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public void registerRPCHook(RPCHook rpcHook) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java index 55b939201561f3645ebb5fff332b397c5d027ed4..ced7c2014f4a5d24b5832971142e686ebc404df0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java @@ -25,7 +25,7 @@ import java.util.Random; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; -import org.apache.rocketmq.broker.topic.TopicValidator; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicFilterType; @@ -176,6 +176,9 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc if (!TopicValidator.validateTopic(requestHeader.getTopic(), response)) { return response; } + if (TopicValidator.isNotAllowedSendTopic(requestHeader.getTopic(), response)) { + return response; + } TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index eb811832058e8343df8aafcbc0e78a96a3106618..97ca51d966d38b18bfdfb430c707a093b2e41a42 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -19,17 +19,6 @@ package org.apache.rocketmq.broker.processor; import com.alibaba.fastjson.JSON; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; -import java.io.UnsupportedEncodingException; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.acl.AccessValidator; import org.apache.rocketmq.acl.plain.PlainAccessValidator; import org.apache.rocketmq.broker.BrokerController; @@ -37,8 +26,8 @@ import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.client.ConsumerGroupInfo; import org.apache.rocketmq.broker.filter.ConsumerFilterData; import org.apache.rocketmq.broker.filter.ExpressionMessageFilter; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; -import org.apache.rocketmq.broker.topic.TopicValidator; import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; @@ -127,6 +116,7 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.ConsumeQueueExt; import org.apache.rocketmq.store.DefaultMessageStore; @@ -137,6 +127,18 @@ import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; +import java.io.UnsupportedEncodingException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; + public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; @@ -234,10 +236,8 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements case RequestCode.GET_BROKER_CLUSTER_ACL_CONFIG: return getBrokerClusterAclConfig(ctx, request); default: - break; + return getUnknownCmdResponse(ctx, request); } - - return null; } @Override @@ -252,29 +252,16 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class); log.info("updateAndCreateTopic called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); - if (requestHeader.getTopic().equals(this.brokerController.getBrokerConfig().getBrokerClusterName())) { - String errorMsg = "the topic[" + requestHeader.getTopic() + "] is conflict with system reserved words."; - log.warn(errorMsg); - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark(errorMsg); - return response; - } + String topic = requestHeader.getTopic(); - if (!TopicValidator.validateTopic(requestHeader.getTopic(), response)) { + if (!TopicValidator.validateTopic(topic, response)) { return response; } - - try { - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - response.markResponseType(); - response.setRemark(null); - ctx.writeAndFlush(response); - } catch (Exception e) { - log.error("Failed to produce a proper response", e); + if (TopicValidator.isSystemTopic(topic, response)) { + return response; } - TopicConfig topicConfig = new TopicConfig(requestHeader.getTopic()); + TopicConfig topicConfig = new TopicConfig(topic); topicConfig.setReadQueueNums(requestHeader.getReadQueueNums()); topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums()); topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum()); @@ -285,7 +272,8 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements this.brokerController.registerIncrementBrokerData(topicConfig, this.brokerController.getTopicConfigManager().getDataVersion()); - return null; + response.setCode(ResponseCode.SUCCESS); + return response; } private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, @@ -296,7 +284,15 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements log.info("deleteTopic called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); - this.brokerController.getTopicConfigManager().deleteTopicConfig(requestHeader.getTopic()); + String topic = requestHeader.getTopic(); + if (!TopicValidator.validateTopic(topic, response)) { + return response; + } + if (TopicValidator.isSystemTopic(topic, response)) { + return response; + } + + this.brokerController.getTopicConfigManager().deleteTopicConfig(topic); this.brokerController.getMessageStore() .cleanUnusedTopic(this.brokerController.getTopicConfigManager().getTopicConfigTable().keySet()); if (this.brokerController.getBrokerConfig().isAutoDeleteUnusedStats()) { @@ -430,7 +426,7 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements responseHeader.setBrokerAddr(this.brokerController.getBrokerAddr()); responseHeader.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); responseHeader.setClusterName(this.brokerController.getBrokerConfig().getBrokerClusterName()); - + response.setCode(ResponseCode.SUCCESS); response.setRemark(null); return response; @@ -462,6 +458,13 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements return null; } + private RemotingCommand getUnknownCmdResponse(ChannelHandlerContext ctx, RemotingCommand request) { + String error = " request type " + request.getCode() + " not supported"; + final RemotingCommand response = + RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error); + return response; + } + private RemotingCommand getAllTopicConfig(ChannelHandlerContext ctx, RemotingCommand request) { final RemotingCommand response = RemotingCommand.createResponseCommand(GetAllTopicConfigResponseHeader.class); // final GetAllTopicConfigResponseHeader responseHeader = @@ -717,6 +720,10 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements this.brokerController.getSubscriptionGroupManager().deleteSubscriptionGroupConfig(requestHeader.getGroupName()); + if (requestHeader.isRemoveOffset()) { + this.brokerController.getConsumerOffsetManager().removeOffset(requestHeader.getGroupName()); + } + if (this.brokerController.getBrokerConfig().isAutoDeleteUnusedStats()) { this.brokerController.getBrokerStatsManager().onGroupDeleted(requestHeader.getGroupName()); } @@ -1118,7 +1125,7 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - Set topics = this.brokerController.getTopicConfigManager().getSystemTopic(); + Set topics = TopicValidator.getSystemTopicSet(); TopicList topicList = new TopicList(); topicList.setTopicList(topics); response.setBody(topicList.encode()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java index b02b5a058838a5eb6305d5f65cd856213536a805..8879a722fcf1967fd013aea6730c48bda6cf5c5c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java @@ -52,6 +52,7 @@ import org.apache.rocketmq.common.protocol.topic.OffsetMovedEvent; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.sysflag.PullSysFlag; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -523,7 +524,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements private void generateOffsetMovedEvent(final OffsetMovedEvent event) { try { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); - msgInner.setTopic(MixAll.OFFSET_MOVED_EVENT); + msgInner.setTopic(TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT); msgInner.setTags(event.getConsumerGroup()); msgInner.setDelayTimeLevel(0); msgInner.setKeys(event.getConsumerGroup()); @@ -583,7 +584,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, channel, request)); } - public void registerConsumeMessageHook(List sendMessageHookList) { - this.consumeMessageHookList = sendMessageHookList; + public void registerConsumeMessageHook(List consumeMessageHookList) { + this.consumeMessageHookList = consumeMessageHookList; } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java index 565857a34779c126154694f0ac6b0881d60d5007..2890fc4db753d922bc2ba7520e6f2f0217ae4f5c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java @@ -251,28 +251,29 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen // Failed case CREATE_MAPEDFILE_FAILED: - log.info("create mapped file failed, server is busy or broken."); + log.warn("create mapped file failed, server is busy or broken."); break; case MESSAGE_ILLEGAL: - log.info( + log.warn( "the message is illegal, maybe msg properties length limit 32k."); break; case PROPERTIES_SIZE_EXCEEDED: - log.info( + log.warn( "the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k."); break; case SERVICE_NOT_AVAILABLE: - log.info( - "service not available now, maybe disk full, maybe your broker machine memory too small."); + log.warn( + "service not available now. It may be caused by one of the following reasons: " + + "the broker's disk is full, messages are put to the slave, message store has been shut down, etc."); break; case OS_PAGECACHE_BUSY: - log.info("[PC_SYNCHRONIZED]broker busy, start flow control for a while"); + log.warn("[PC_SYNCHRONIZED]broker busy, start flow control for a while"); break; case UNKNOWN_ERROR: - log.info("UNKNOWN_ERROR"); + log.warn("UNKNOWN_ERROR"); break; default: - log.info("UNKNOWN_ERROR DEFAULT"); + log.warn("UNKNOWN_ERROR DEFAULT"); break; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 20b5cfaae829351d6d89496540f9bd5bb71d14f1..724cf54c8137c57d1dd8a5d434e20e582674d7a7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -218,6 +218,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement String originMsgId = MessageAccessor.getOriginMessageId(msgExt); MessageAccessor.setOriginMessageId(msgInner, UtilAll.isBlank(originMsgId) ? msgExt.getMsgId() : originMsgId); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + CompletableFuture putMessageResult = this.brokerController.getMessageStore().asyncPutMessage(msgInner); return putMessageResult.thenApply((r) -> { if (r != null) { @@ -480,7 +482,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement case SERVICE_NOT_AVAILABLE: response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); response.setRemark( - "service not available now, maybe disk full, " + diskUtil() + ", maybe your broker machine memory too small."); + "service not available now. It may be caused by one of the following reasons: " + + "the broker's disk is full [" + diskUtil() + "], messages are put to the slave, message store has been shut down, etc."); break; case OS_PAGECACHE_BUSY: response.setCode(ResponseCode.SYSTEM_ERROR); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 199b46d603a72ffb4561d21a6e146df291228390..1f2bb4d8ba49c00a4b10a0f674bd05aaf9aaa7a2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.broker.topic; -import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; @@ -37,18 +36,20 @@ import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.sysflag.TopicSysFlag; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; public class TopicConfigManager extends ConfigManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final long LOCK_TIMEOUT_MILLIS = 3000; - private transient final Lock lockTopicConfigTable = new ReentrantLock(); + private static final int SCHEDULE_TOPIC_QUEUE_NUM = 18; + + private transient final Lock topicConfigTableLock = new ReentrantLock(); private final ConcurrentMap topicConfigTable = new ConcurrentHashMap(1024); private final DataVersion dataVersion = new DataVersion(); - private final Set systemTopicList = new HashSet(); private transient BrokerController brokerController; public TopicConfigManager() { @@ -57,20 +58,18 @@ public class TopicConfigManager extends ConfigManager { public TopicConfigManager(BrokerController brokerController) { this.brokerController = brokerController; { - // MixAll.SELF_TEST_TOPIC - String topic = MixAll.SELF_TEST_TOPIC; + String topic = TopicValidator.RMQ_SYS_SELF_TEST_TOPIC; TopicConfig topicConfig = new TopicConfig(topic); - this.systemTopicList.add(topic); + TopicValidator.addSystemTopic(topic); topicConfig.setReadQueueNums(1); topicConfig.setWriteQueueNums(1); this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); } { - // MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC if (this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) { - String topic = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC; + String topic = TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC; TopicConfig topicConfig = new TopicConfig(topic); - this.systemTopicList.add(topic); + TopicValidator.addSystemTopic(topic); topicConfig.setReadQueueNums(this.brokerController.getBrokerConfig() .getDefaultTopicQueueNums()); topicConfig.setWriteQueueNums(this.brokerController.getBrokerConfig() @@ -81,10 +80,9 @@ public class TopicConfigManager extends ConfigManager { } } { - // MixAll.BENCHMARK_TOPIC - String topic = MixAll.BENCHMARK_TOPIC; + String topic = TopicValidator.RMQ_SYS_BENCHMARK_TOPIC; TopicConfig topicConfig = new TopicConfig(topic); - this.systemTopicList.add(topic); + TopicValidator.addSystemTopic(topic); topicConfig.setReadQueueNums(1024); topicConfig.setWriteQueueNums(1024); this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); @@ -93,7 +91,7 @@ public class TopicConfigManager extends ConfigManager { String topic = this.brokerController.getBrokerConfig().getBrokerClusterName(); TopicConfig topicConfig = new TopicConfig(topic); - this.systemTopicList.add(topic); + TopicValidator.addSystemTopic(topic); int perm = PermName.PERM_INHERIT; if (this.brokerController.getBrokerConfig().isClusterTopicEnable()) { perm |= PermName.PERM_READ | PermName.PERM_WRITE; @@ -105,7 +103,7 @@ public class TopicConfigManager extends ConfigManager { String topic = this.brokerController.getBrokerConfig().getBrokerName(); TopicConfig topicConfig = new TopicConfig(topic); - this.systemTopicList.add(topic); + TopicValidator.addSystemTopic(topic); int perm = PermName.PERM_INHERIT; if (this.brokerController.getBrokerConfig().isBrokerTopicEnable()) { perm |= PermName.PERM_READ | PermName.PERM_WRITE; @@ -116,19 +114,26 @@ public class TopicConfigManager extends ConfigManager { this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); } { - // MixAll.OFFSET_MOVED_EVENT - String topic = MixAll.OFFSET_MOVED_EVENT; + String topic = TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT; TopicConfig topicConfig = new TopicConfig(topic); - this.systemTopicList.add(topic); + TopicValidator.addSystemTopic(topic); topicConfig.setReadQueueNums(1); topicConfig.setWriteQueueNums(1); this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); } + { + String topic = TopicValidator.RMQ_SYS_SCHEDULE_TOPIC; + TopicConfig topicConfig = new TopicConfig(topic); + TopicValidator.addSystemTopic(topic); + topicConfig.setReadQueueNums(SCHEDULE_TOPIC_QUEUE_NUM); + topicConfig.setWriteQueueNums(SCHEDULE_TOPIC_QUEUE_NUM); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } { if (this.brokerController.getBrokerConfig().isTraceTopicEnable()) { String topic = this.brokerController.getBrokerConfig().getMsgTraceTopicName(); TopicConfig topicConfig = new TopicConfig(topic); - this.systemTopicList.add(topic); + TopicValidator.addSystemTopic(topic); topicConfig.setReadQueueNums(1); topicConfig.setWriteQueueNums(1); this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); @@ -137,21 +142,13 @@ public class TopicConfigManager extends ConfigManager { { String topic = this.brokerController.getBrokerConfig().getBrokerClusterName() + "_" + MixAll.REPLY_TOPIC_POSTFIX; TopicConfig topicConfig = new TopicConfig(topic); - this.systemTopicList.add(topic); + TopicValidator.addSystemTopic(topic); topicConfig.setReadQueueNums(1); topicConfig.setWriteQueueNums(1); this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); } } - public boolean isSystemTopic(final String topic) { - return this.systemTopicList.contains(topic); - } - - public Set getSystemTopic() { - return this.systemTopicList; - } - public TopicConfig selectTopicConfig(final String topic) { return this.topicConfigTable.get(topic); } @@ -162,7 +159,7 @@ public class TopicConfigManager extends ConfigManager { boolean createNew = false; try { - if (this.lockTopicConfigTable.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + if (this.topicConfigTableLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { try { topicConfig = this.topicConfigTable.get(topic); if (topicConfig != null) @@ -170,7 +167,7 @@ public class TopicConfigManager extends ConfigManager { TopicConfig defaultTopicConfig = this.topicConfigTable.get(defaultTopic); if (defaultTopicConfig != null) { - if (defaultTopic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { + if (defaultTopic.equals(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC)) { if (!this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) { defaultTopicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE); } @@ -179,9 +176,7 @@ public class TopicConfigManager extends ConfigManager { if (PermName.isInherited(defaultTopicConfig.getPerm())) { topicConfig = new TopicConfig(topic); - int queueNums = - clientDefaultTopicQueueNums > defaultTopicConfig.getWriteQueueNums() ? defaultTopicConfig - .getWriteQueueNums() : clientDefaultTopicQueueNums; + int queueNums = Math.min(clientDefaultTopicQueueNums, defaultTopicConfig.getWriteQueueNums()); if (queueNums < 0) { queueNums = 0; @@ -216,7 +211,7 @@ public class TopicConfigManager extends ConfigManager { this.persist(); } } finally { - this.lockTopicConfigTable.unlock(); + this.topicConfigTableLock.unlock(); } } } catch (InterruptedException e) { @@ -242,7 +237,7 @@ public class TopicConfigManager extends ConfigManager { boolean createNew = false; try { - if (this.lockTopicConfigTable.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + if (this.topicConfigTableLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { try { topicConfig = this.topicConfigTable.get(topic); if (topicConfig != null) @@ -260,7 +255,7 @@ public class TopicConfigManager extends ConfigManager { this.dataVersion.nextVersion(); this.persist(); } finally { - this.lockTopicConfigTable.unlock(); + this.topicConfigTableLock.unlock(); } } } catch (InterruptedException e) { @@ -275,32 +270,32 @@ public class TopicConfigManager extends ConfigManager { } public TopicConfig createTopicOfTranCheckMaxTime(final int clientDefaultTopicQueueNums, final int perm) { - TopicConfig topicConfig = this.topicConfigTable.get(MixAll.TRANS_CHECK_MAX_TIME_TOPIC); + TopicConfig topicConfig = this.topicConfigTable.get(TopicValidator.RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC); if (topicConfig != null) return topicConfig; boolean createNew = false; try { - if (this.lockTopicConfigTable.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + if (this.topicConfigTableLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { try { - topicConfig = this.topicConfigTable.get(MixAll.TRANS_CHECK_MAX_TIME_TOPIC); + topicConfig = this.topicConfigTable.get(TopicValidator.RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC); if (topicConfig != null) return topicConfig; - topicConfig = new TopicConfig(MixAll.TRANS_CHECK_MAX_TIME_TOPIC); + topicConfig = new TopicConfig(TopicValidator.RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC); topicConfig.setReadQueueNums(clientDefaultTopicQueueNums); topicConfig.setWriteQueueNums(clientDefaultTopicQueueNums); topicConfig.setPerm(perm); topicConfig.setTopicSysFlag(0); log.info("create new topic {}", topicConfig); - this.topicConfigTable.put(MixAll.TRANS_CHECK_MAX_TIME_TOPIC, topicConfig); + this.topicConfigTable.put(TopicValidator.RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC, topicConfig); createNew = true; this.dataVersion.nextVersion(); this.persist(); } finally { - this.lockTopicConfigTable.unlock(); + this.topicConfigTableLock.unlock(); } } } catch (InterruptedException e) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicValidator.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicValidator.java deleted file mode 100644 index 58b1cc86fcdfcdbf38d3c3f9656008fae052b8c6..0000000000000000000000000000000000000000 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicValidator.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.broker.topic; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.protocol.ResponseCode; -import org.apache.rocketmq.remoting.protocol.RemotingCommand; - -public class TopicValidator { - - private static final String VALID_PATTERN_STR = "^[%|a-zA-Z0-9_-]+$"; - private static final Pattern PATTERN = Pattern.compile(VALID_PATTERN_STR); - private static final int TOPIC_MAX_LENGTH = 127; - - private static boolean regularExpressionMatcher(String origin, Pattern pattern) { - if (pattern == null) { - return true; - } - Matcher matcher = pattern.matcher(origin); - return matcher.matches(); - } - - public static boolean validateTopic(String topic, RemotingCommand response) { - - if (UtilAll.isBlank(topic)) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("The specified topic is blank."); - return false; - } - - if (!regularExpressionMatcher(topic, PATTERN)) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("The specified topic contains illegal characters, allowing only " + VALID_PATTERN_STR); - return false; - } - - if (topic.length() > TOPIC_MAX_LENGTH) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("The specified topic is longer than topic max length."); - return false; - } - - //whether the same with system reserved keyword - if (topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("The specified topic is conflict with AUTO_CREATE_TOPIC_KEY_TOPIC."); - return false; - } - - return true; - } -} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java index 35d811207ba6687949877daa251730c3f26c77f6..4cf5647084da6f537dd60769ee980805acb7cffa 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java @@ -69,7 +69,7 @@ public abstract class AbstractTransactionalMessageCheckListener { msgExt.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID))); msgExt.setStoreSize(0); String groupId = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); - Channel channel = brokerController.getProducerManager().getAvaliableChannel(groupId); + Channel channel = brokerController.getProducerManager().getAvailableChannel(groupId); if (channel != null) { brokerController.getBroker2Client().checkProducerTransactionState(groupId, channel, checkTransactionStateRequestHeader, msgExt); } else { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index 25065ebe885f5644aa128f96e09a6583ad0ede65..74df47a088eab5bfcb41daeac6de8582c3fe568e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -21,13 +21,13 @@ import org.apache.rocketmq.broker.transaction.OperationResult; import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.MessageExtBrokerInner; @@ -127,7 +127,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener) { try { - String topic = MixAll.RMQ_SYS_TRANS_HALF_TOPIC; + String topic = TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC; Set msgQueues = transactionalMessageBridge.fetchMessageQueues(topic); if (msgQueues == null || msgQueues.size() == 0) { log.warn("The queue of topic is empty :" + topic); @@ -164,7 +164,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ break; } if (removeMap.containsKey(i)) { - log.info("Half offset {} has been committed/rolled back", i); + log.debug("Half offset {} has been committed/rolled back", i); Long removedOpOffset = removeMap.remove(i); doneOpOffset.add(removedOpOffset); } else { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java index 3042b4c3e568479407178072bae798a8751a76cd..e6baf0266ccf431a249514b91565092f844541c9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageUtil.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.broker.transaction.queue; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.topic.TopicValidator; import java.nio.charset.Charset; @@ -25,11 +26,11 @@ public class TransactionalMessageUtil { public static Charset charset = Charset.forName("utf-8"); public static String buildOpTopic() { - return MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC; + return TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC; } public static String buildHalfTopic() { - return MixAll.RMQ_SYS_TRANS_HALF_TOPIC; + return TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC; } public static String buildConsumerGroup() { diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java index 68d58ef4400ffb5028d8c1b667dcf8ddfc236d22..9ea1eeee31b9995f35890a8b06e8ff765690b8ef 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java @@ -124,7 +124,7 @@ public class BrokerOuterAPITest { boolean success = Iterables.any(booleanList, new Predicate() { public boolean apply(Boolean input) { - return input ? true : false; + return input; } }); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java index 4791ab1f053be7fe648309e8cc0d84c35dcc0c82..6c794ac5d33904ba10a7d5a71ec3db22731787a7 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java @@ -110,20 +110,20 @@ public class ProducerManagerTest { } @Test - public void testGetAvaliableChannel() { + public void testGetAvailableChannel() { producerManager.registerProducer(group, clientInfo); when(channel.isActive()).thenReturn(true); when(channel.isWritable()).thenReturn(true); - Channel c = producerManager.getAvaliableChannel(group); + Channel c = producerManager.getAvailableChannel(group); assertThat(c).isSameAs(channel); when(channel.isWritable()).thenReturn(false); - c = producerManager.getAvaliableChannel(group); + c = producerManager.getAvailableChannel(group); assertThat(c).isSameAs(channel); when(channel.isActive()).thenReturn(false); - c = producerManager.getAvaliableChannel(group); + c = producerManager.getAvailableChannel(group); assertThat(c).isNull(); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java index ec0a879c209c97d7c8ab2f63f254e343b0b62fba..2764761d39f68fa081a961479a02ce25825a4f2d 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java @@ -16,17 +16,22 @@ */ package org.apache.rocketmq.broker.processor; +import com.google.common.collect.Sets; import io.netty.channel.ChannelHandlerContext; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicFilterType; +import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader; +import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; @@ -47,6 +52,10 @@ import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.Set; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -61,17 +70,30 @@ public class AdminBrokerProcessorTest { @Spy private BrokerController - brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), - new MessageStoreConfig()); + brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), + new MessageStoreConfig()); @Mock private MessageStore messageStore; + private Set systemTopicSet; @Before public void init() { brokerController.setMessageStore(messageStore); adminBrokerProcessor = new AdminBrokerProcessor(brokerController); + + systemTopicSet = Sets.newHashSet( + TopicValidator.RMQ_SYS_SELF_TEST_TOPIC, + TopicValidator.RMQ_SYS_BENCHMARK_TOPIC, + TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, + TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT, + TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC, + this.brokerController.getBrokerConfig().getBrokerClusterName(), + this.brokerController.getBrokerConfig().getBrokerClusterName() + "_" + MixAll.REPLY_TOPIC_POSTFIX); + if (this.brokerController.getBrokerConfig().isTraceTopicEnable()) { + systemTopicSet.add(this.brokerController.getBrokerConfig().getMsgTraceTopicName()); + } } @Test @@ -94,6 +116,67 @@ public class AdminBrokerProcessorTest { assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); } + @Test + public void testUpdateAndCreateTopic() throws Exception { + //test system topic + for (String topic : systemTopicSet) { + RemotingCommand request = buildCreateTopicRequest(topic); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).isEqualTo("The topic[" + topic + "] is conflict with system topic."); + } + + //test validate error topic + String topic = ""; + RemotingCommand request = buildCreateTopicRequest(topic); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + topic = "TEST_CREATE_TOPIC"; + request = buildCreateTopicRequest(topic); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + } + + @Test + public void testDeleteTopic() throws Exception { + //test system topic + for (String topic : systemTopicSet) { + RemotingCommand request = buildDeleteTopicRequest(topic); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).isEqualTo("The topic[" + topic + "] is conflict with system topic."); + } + + String topic = "TEST_DELETE_TOPIC"; + RemotingCommand request = buildDeleteTopicRequest(topic); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + private RemotingCommand buildCreateTopicRequest(String topic) { + CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setTopicFilterType(TopicFilterType.SINGLE_TAG.name()); + requestHeader.setReadQueueNums(8); + requestHeader.setWriteQueueNums(8); + requestHeader.setPerm(PermName.PERM_READ | PermName.PERM_WRITE); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader); + request.makeCustomHeaderToNet(); + return request; + } + + private RemotingCommand buildDeleteTopicRequest(String topic) { + DeleteTopicRequestHeader requestHeader = new DeleteTopicRequestHeader(); + requestHeader.setTopic(topic); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_BROKER, requestHeader); + request.makeCustomHeaderToNet(); + return request; + } + private MessageExt createDefaultMessageExt() { MessageExt messageExt = new MessageExt(); messageExt.setMsgId("12345678"); @@ -106,10 +189,11 @@ public class AdminBrokerProcessorTest { return messageExt; } - private SelectMappedBufferResult createSelectMappedBufferResult(){ - SelectMappedBufferResult result = new SelectMappedBufferResult(0, ByteBuffer.allocate(1024) ,0, new MappedFile()); + private SelectMappedBufferResult createSelectMappedBufferResult() { + SelectMappedBufferResult result = new SelectMappedBufferResult(0, ByteBuffer.allocate(1024), 0, new MappedFile()); return result; } + private ResumeCheckHalfMessageRequestHeader createResumeCheckHalfMessageRequestHeader() { ResumeCheckHalfMessageRequestHeader header = new ResumeCheckHalfMessageRequestHeader(); header.setMsgId("C0A803CA00002A9F0000000000031367"); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java index 85c775040f53ef6093e979c8c6adc53a3a36161f..eeca6e7622c5d5e37bc7be493adcc9df668e5eab 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java @@ -25,15 +25,14 @@ import java.util.Map; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.client.net.Broker2Client; -import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.common.BrokerConfig; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; @@ -115,7 +114,7 @@ public class ReplyMessageProcessorTest { SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); requestHeader.setProducerGroup(group); requestHeader.setTopic(topic); - requestHeader.setDefaultTopic(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC); + requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC); requestHeader.setDefaultTopicQueueNums(3); requestHeader.setQueueId(1); requestHeader.setSysFlag(0); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java index bdf13d4a7faa7334861df540d9e5910996ad4b0a..b9344e90ed600f29ac72c2720b60fbbff1cda5d4 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java @@ -23,7 +23,6 @@ import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.common.BrokerConfig; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; @@ -32,6 +31,7 @@ import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; @@ -238,7 +238,7 @@ public class SendMessageProcessorTest { SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); requestHeader.setProducerGroup(group); requestHeader.setTopic(topic); - requestHeader.setDefaultTopic(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC); + requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC); requestHeader.setDefaultTopicQueueNums(3); requestHeader.setQueueId(1); requestHeader.setSysFlag(0); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java index 653a96933ed5decb6b19a0086008c96d90ad7c77..f6035463c01a4588f69f7094d02a331416284b5a 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListenerTest.java @@ -19,10 +19,10 @@ package org.apache.rocketmq.broker.transaction.queue; import java.net.InetSocketAddress; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.BrokerConfig; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.store.MessageExtBrokerInner; @@ -92,7 +92,7 @@ public class DefaultTransactionalMessageCheckListenerTest { @Test public void testResolveDiscardMsg() { MessageExt messageExt = new MessageExt(); - messageExt.setTopic(MixAll.RMQ_SYS_TRANS_HALF_TOPIC); + messageExt.setTopic(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC); messageExt.setQueueId(0); messageExt.setBody("test resolve discard msg".getBytes()); messageExt.setStoreHost(new InetSocketAddress("127.0.0.1", 10911)); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java index 5d8c2b955d6cadac46eb8302a7d79bcaeb6951fa..031a55a5d9d74af1151896834d8f5d8656084894 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java @@ -20,11 +20,11 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.common.BrokerConfig; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.store.AppendMessageResult; @@ -98,7 +98,7 @@ public class TransactionalMessageBridgeTest { @Test public void testFetchMessageQueues() { - Set messageQueues = transactionBridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC); + Set messageQueues = transactionBridge.fetchMessageQueues(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC); assertThat(messageQueues.size()).isEqualTo(1); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java index 47eccbe026ee5f1e9a04092cc90d1af7f2e512b8..8b138fc896a5e929fbfa8f5e1f9b91d9cc139a98 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImplTest.java @@ -23,13 +23,13 @@ import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.common.BrokerConfig; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.store.AppendMessageResult; @@ -108,10 +108,10 @@ public class TransactionalMessageServiceImplTest { @Test public void testCheck_withDiscard() { - when(bridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)); - when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createDiscardPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 5, "hellp", 1)); - when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 6, "hellp", 0)); - when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createOpPulResult(MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, "10", 1)); + when(bridge.fetchMessageQueues(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC)); + when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createDiscardPullResult(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC, 5, "hellp", 1)); + when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC, 6, "hellp", 0)); + when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createOpPulResult(TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, "10", 1)); long timeOut = this.brokerController.getBrokerConfig().getTransactionTimeOut(); int checkMax = this.brokerController.getBrokerConfig().getTransactionCheckMax(); final AtomicInteger checkMessage = new AtomicInteger(0); @@ -128,10 +128,10 @@ public class TransactionalMessageServiceImplTest { @Test public void testCheck_withCheck() { - when(bridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)); - when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 5, "hello", 1)); - when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 6, "hellp", 0)); - when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, "5", 0)); + when(bridge.fetchMessageQueues(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC)); + when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createPullResult(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC, 5, "hello", 1)); + when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC, 6, "hellp", 0)); + when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createPullResult(TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, "5", 0)); when(bridge.getBrokerController()).thenReturn(this.brokerController); when(bridge.renewHalfMessageInner(any(MessageExtBrokerInner.class))).thenReturn(createMessageBrokerInner()); when(bridge.putMessageReturnResult(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult diff --git a/client/pom.xml b/client/pom.xml index b73b6b697b46cdff5288a104e46f17bcfa021873..95ef4617dd1b68b56cbebc6fab990ea783c8d602 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -27,11 +27,6 @@ rocketmq-client rocketmq-client ${project.version} - - 1.6 - 1.6 - - ${project.groupId} @@ -52,6 +47,22 @@ org.apache.commons commons-lang3 + + commons-codec + commons-codec + + + io.opentracing + opentracing-api + 0.33.0 + provided + + + io.opentracing + opentracing-mock + 0.33.0 + test + org.apache.logging.log4j log4j-core diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java index d0ae5e1b8315370c3ce3373c330e9a7d0df2ca84..3ab214b8db6437f21ab13b0fb615bbaec89409e7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java @@ -94,7 +94,7 @@ public class ClientConfig { public void changeInstanceNameToPID() { if (this.instanceName.equals("DEFAULT")) { - this.instanceName = String.valueOf(UtilAll.getPid()); + this.instanceName = UtilAll.getPid() + "#" + System.nanoTime(); } } @@ -178,8 +178,8 @@ public class ClientConfig { } public String getNamesrvAddr() { - if (StringUtils.isNotEmpty(namesrvAddr) && NameServerAddressUtils.NAMESRV_ENDPOINT_PATTERN.matcher(namesrvAddr.trim()).matches()) { - return namesrvAddr.substring(NameServerAddressUtils.ENDPOINT_PREFIX.length()); + if (StringUtils.isNotEmpty(namesrvAddr) && NameServerAddressUtils.validateInstanceEndpoint(namesrvAddr.trim())) { + return NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(namesrvAddr); } return namesrvAddr; } diff --git a/client/src/main/java/org/apache/rocketmq/client/Validators.java b/client/src/main/java/org/apache/rocketmq/client/Validators.java index d77faf3c222dbcc828c3182ceb4301ba25072d10..e712e2f5e46a2c34e9f21eea73790e0ec8d5d126 100644 --- a/client/src/main/java/org/apache/rocketmq/client/Validators.java +++ b/client/src/main/java/org/apache/rocketmq/client/Validators.java @@ -21,10 +21,10 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.topic.TopicValidator; /** * Common Validator @@ -85,6 +85,7 @@ public class Validators { } // topic Validators.checkTopic(msg.getTopic()); + Validators.isNotAllowedSendTopic(msg.getTopic()); // body if (null == msg.getBody()) { @@ -116,11 +117,19 @@ public class Validators { throw new MQClientException( String.format("The specified topic is longer than topic max length %d.", TOPIC_MAX_LENGTH), null); } + } + + public static void isSystemTopic(String topic) throws MQClientException { + if (TopicValidator.isSystemTopic(topic)) { + throw new MQClientException( + String.format("The topic[%s] is conflict with system topic.", topic), null); + } + } - //whether the same with system reserved keyword - if (topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { + public static void isNotAllowedSendTopic(String topic) throws MQClientException { + if (TopicValidator.isNotAllowedSendTopic(topic)) { throw new MQClientException( - String.format("The topic[%s] is conflict with AUTO_CREATE_TOPIC_KEY_TOPIC.", topic), null); + String.format("Sending message to topic[%s] is forbidden.", topic), null); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java index ab223c3ecac23c373716c4b2f5844c404328c892..891c17e3ba2c05a06816ada0502548ef32b4e9e2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java +++ b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java @@ -23,21 +23,15 @@ public class ThreadLocalIndex { private final ThreadLocal threadLocalIndex = new ThreadLocal(); private final Random random = new Random(); - public int getAndIncrement() { + public int incrementAndGet() { Integer index = this.threadLocalIndex.get(); if (null == index) { index = Math.abs(random.nextInt()); - if (index < 0) - index = 0; this.threadLocalIndex.set(index); } - index = Math.abs(index + 1); - if (index < 0) - index = 0; - - this.threadLocalIndex.set(index); - return index; + this.threadLocalIndex.set(++index); + return Math.abs(index); } @Override diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 6718eb55fd8730ac9ff0187f4f0b54fc48b932fa..c54399aa5ed24346fc801e4d9b15050ee7a2b997 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -23,6 +23,10 @@ import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAverage import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; +import org.apache.rocketmq.client.trace.TraceDispatcher; +import org.apache.rocketmq.client.trace.hook.ConsumeMessageTraceHookImpl; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; @@ -30,10 +34,13 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.NamespaceUtil; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; public class DefaultLitePullConsumer extends ClientConfig implements LitePullConsumer { + private final InternalLogger log = ClientLogger.getLog(); + private final DefaultLitePullConsumerImpl defaultLitePullConsumerImpl; /** @@ -153,6 +160,21 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon */ private String consumeTimestamp = UtilAll.timeMillisToHumanString3(System.currentTimeMillis() - (1000 * 60 * 30)); + /** + * Interface of asynchronous transfer data + */ + private TraceDispatcher traceDispatcher = null; + + /** + * The flag for message trace + */ + private boolean enableMsgTrace = false; + + /** + * The name value of message trace topic.If you don't config,you can use the default trace topic name. + */ + private String customizedTraceTopic; + /** * Default constructor. */ @@ -202,13 +224,29 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon @Override public void start() throws MQClientException { + setTraceDispatcher(); setConsumerGroup(NamespaceUtil.wrapNamespace(this.getNamespace(), this.consumerGroup)); this.defaultLitePullConsumerImpl.start(); + if (null != traceDispatcher) { + try { + traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel()); + } catch (MQClientException e) { + log.warn("trace dispatcher start failed ", e); + } + } } @Override public void shutdown() { this.defaultLitePullConsumerImpl.shutdown(); + if (null != traceDispatcher) { + traceDispatcher.shutdown(); + } + } + + @Override + public boolean isRunning() { + return this.defaultLitePullConsumerImpl.isRunning(); } @Override @@ -307,6 +345,22 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon this.autoCommit = autoCommit; } + public boolean isConnectBrokerByUser() { + return this.defaultLitePullConsumerImpl.getPullAPIWrapper().isConnectBrokerByUser(); + } + + public void setConnectBrokerByUser(boolean connectBrokerByUser) { + this.defaultLitePullConsumerImpl.getPullAPIWrapper().setConnectBrokerByUser(connectBrokerByUser); + } + + public long getDefaultBrokerId() { + return this.defaultLitePullConsumerImpl.getPullAPIWrapper().getDefaultBrokerId(); + } + + public void setDefaultBrokerId(long defaultBrokerId) { + this.defaultLitePullConsumerImpl.getPullAPIWrapper().setDefaultBrokerId(defaultBrokerId); + } + public int getPullThreadNums() { return pullThreadNums; } @@ -469,4 +523,36 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon public void setConsumeTimestamp(String consumeTimestamp) { this.consumeTimestamp = consumeTimestamp; } + + public TraceDispatcher getTraceDispatcher() { + return traceDispatcher; + } + + public void setCustomizedTraceTopic(String customizedTraceTopic) { + this.customizedTraceTopic = customizedTraceTopic; + } + + private void setTraceDispatcher() { + if (isEnableMsgTrace()) { + try { + this.traceDispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, customizedTraceTopic, null); + this.defaultLitePullConsumerImpl.registerConsumeMessageHook( + new ConsumeMessageTraceHookImpl(traceDispatcher)); + } catch (Throwable e) { + log.error("system mqtrace hook init failed ,maybe can't send msg trace data"); + } + } + } + + public String getCustomizedTraceTopic() { + return customizedTraceTopic; + } + + public boolean isEnableMsgTrace() { + return enableMsgTrace; + } + + public void setEnableMsgTrace(boolean enableMsgTrace) { + this.enableMsgTrace = enableMsgTrace; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index 6ad0fc308ecfbf11951a4f56ac778c1f58a289ec..9011117a79fc247800c3c9aa06d58c5c97bb01a8 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -254,6 +254,11 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume */ private long consumeTimeout = 15; + /** + * Maximum time to await message consuming when shutdown consumer, 0 indicates no await. + */ + private long awaitTerminationMillisWhenShutdown = 0; + /** * Interface of asynchronous transfer data */ @@ -705,7 +710,7 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume */ @Override public void shutdown() { - this.defaultMQPushConsumerImpl.shutdown(); + this.defaultMQPushConsumerImpl.shutdown(awaitTerminationMillisWhenShutdown); if (null != traceDispatcher) { traceDispatcher.shutdown(); } @@ -886,6 +891,14 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume this.consumeTimeout = consumeTimeout; } + public long getAwaitTerminationMillisWhenShutdown() { + return awaitTerminationMillisWhenShutdown; + } + + public void setAwaitTerminationMillisWhenShutdown(long awaitTerminationMillisWhenShutdown) { + this.awaitTerminationMillisWhenShutdown = awaitTerminationMillisWhenShutdown; + } + public TraceDispatcher getTraceDispatcher() { return traceDispatcher; } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java index ce2228803383d7fc1502b0fa5babd9bd1ec5c0ac..25b11046f84d7e98ecadd1b7cf8af6ae80625107 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java @@ -35,6 +35,13 @@ public interface LitePullConsumer { */ void shutdown(); + /** + * This consumer is still running + * + * @return true if consumer is still running + */ + boolean isRunning(); + /** * Subscribe some topic with subExpression * diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java index 37568317cb016638ef5d56c23341051810ce8ec9..31f4e3522ec6e2a769f427f27118d5d616ca33a9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java @@ -49,7 +49,7 @@ public class AllocateMessageQueueByMachineRoom implements AllocateMessageQueueSt int startIndex = mod * currentIndex; int endIndex = startIndex + mod; for (int i = startIndex; i < endIndex; i++) { - result.add(mqAll.get(i)); + result.add(premqAll.get(i)); } if (rem > currentIndex) { result.add(premqAll.get(currentIndex + mod * cidAll.size())); diff --git a/client/src/main/java/org/apache/rocketmq/client/exception/MQBrokerException.java b/client/src/main/java/org/apache/rocketmq/client/exception/MQBrokerException.java index e4f2c8df85022b66f6345cd7c28edff8af44d6aa..f07a38b81f0a3689015eed3ac63f6f2fb12cb457 100644 --- a/client/src/main/java/org/apache/rocketmq/client/exception/MQBrokerException.java +++ b/client/src/main/java/org/apache/rocketmq/client/exception/MQBrokerException.java @@ -23,12 +23,22 @@ public class MQBrokerException extends Exception { private static final long serialVersionUID = 5975020272601250368L; private final int responseCode; private final String errorMessage; + private final String brokerAddr; public MQBrokerException(int responseCode, String errorMessage) { super(FAQUrl.attachDefaultURL("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " - + errorMessage)); + + errorMessage)); this.responseCode = responseCode; this.errorMessage = errorMessage; + this.brokerAddr = null; + } + + public MQBrokerException(int responseCode, String errorMessage, String brokerAddr) { + super(FAQUrl.attachDefaultURL("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " + + errorMessage + (brokerAddr != null ? " BROKER: " + brokerAddr : ""))); + this.responseCode = responseCode; + this.errorMessage = errorMessage; + this.brokerAddr = brokerAddr; } public int getResponseCode() { @@ -38,4 +48,8 @@ public class MQBrokerException extends Exception { public String getErrorMessage() { return errorMessage; } + + public String getBrokerAddr() { + return brokerAddr; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/hook/EndTransactionContext.java b/client/src/main/java/org/apache/rocketmq/client/hook/EndTransactionContext.java new file mode 100644 index 0000000000000000000000000000000000000000..5271ade82bff79cacb6c5d9e822003c80cfaf503 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/hook/EndTransactionContext.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.hook; + +import org.apache.rocketmq.client.producer.LocalTransactionState; +import org.apache.rocketmq.common.message.Message; + +public class EndTransactionContext { + private String producerGroup; + private Message message; + private String brokerAddr; + private String msgId; + private String transactionId; + private LocalTransactionState transactionState; + private boolean fromTransactionCheck; + + public String getProducerGroup() { + return producerGroup; + } + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + public Message getMessage() { + return message; + } + + public void setMessage(Message message) { + this.message = message; + } + + public String getBrokerAddr() { + return brokerAddr; + } + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + public String getMsgId() { + return msgId; + } + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + public String getTransactionId() { + return transactionId; + } + + public void setTransactionId(String transactionId) { + this.transactionId = transactionId; + } + + public LocalTransactionState getTransactionState() { + return transactionState; + } + + public void setTransactionState(LocalTransactionState transactionState) { + this.transactionState = transactionState; + } + + public boolean isFromTransactionCheck() { + return fromTransactionCheck; + } + + public void setFromTransactionCheck(boolean fromTransactionCheck) { + this.fromTransactionCheck = fromTransactionCheck; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/hook/EndTransactionHook.java b/client/src/main/java/org/apache/rocketmq/client/hook/EndTransactionHook.java new file mode 100644 index 0000000000000000000000000000000000000000..834cb2731264b150e6ad52f6c5be74ba5e6087c3 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/hook/EndTransactionHook.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.hook; + +public interface EndTransactionHook { + String hookName(); + + void endTransaction(final EndTransactionContext context); +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index 3ef60af1799a9c91664581f8b596bb4046c6da1b..a541b2d379c6d6eea857240978e759801543fb99 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -282,8 +282,6 @@ public class ClientRemotingProcessor extends AsyncNettyRequestProcessor implemen if (requestResponseFuture.getRequestCallback() != null) { requestResponseFuture.getRequestCallback().onSuccess(replyMsg); - } else { - requestResponseFuture.putResponseMessage(replyMsg); } } else { String bornHost = replyMsg.getBornHostString(); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index 9dbd55201dc6bea8ff1046e59be1fbd674a2b63b..8884e4adfef9efd5caef99845d9e1ba4cd7c4da1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -82,6 +82,7 @@ public class MQAdminImpl { public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { try { Validators.checkTopic(newTopic); + Validators.isSystemTopic(newTopic); TopicRouteData topicRouteData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(key, timeoutMillis); List brokerDataList = topicRouteData.getBrokerDatas(); if (brokerDataList != null && !brokerDataList.isEmpty()) { @@ -267,17 +268,23 @@ public class MQAdminImpl { messageId.getOffset(), timeoutMillis); } - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, - long end) throws MQClientException, - InterruptedException { + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return queryMessage(topic, key, maxNum, begin, end, false); } + public QueryResult queryMessageByUniqKey(String topic, String uniqKey, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + + return queryMessage(topic, uniqKey, maxNum, begin, end, true); + } + public MessageExt queryMessageByUniqKey(String topic, String uniqKey) throws InterruptedException, MQClientException { - QueryResult qr = this.queryMessage(topic, uniqKey, 32, - MessageClientIDSetter.getNearlyTimeFromID(uniqKey).getTime() - 1000, Long.MAX_VALUE, true); + QueryResult qr = queryMessageByUniqKey(topic, uniqKey, 32, + MessageClientIDSetter.getNearlyTimeFromID(uniqKey).getTime() - 1000, Long.MAX_VALUE); if (qr != null && qr.getMessageList() != null && qr.getMessageList().size() > 0) { return qr.getMessageList().get(0); } else { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 073e3679be0462ed657a942c1a63ba0698f04931..63b2045d1d5d5665217ac80c106cd703205c53eb 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -390,7 +390,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } @@ -414,7 +414,7 @@ public class MQClientAPIImpl { default: break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } @@ -502,7 +502,7 @@ public class MQClientAPIImpl { ) throws RemotingException, MQBrokerException, InterruptedException { RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); assert response != null; - return this.processSendResponse(brokerName, msg, response); + return this.processSendResponse(brokerName, msg, response,addr); } private void sendMessageAsync( @@ -528,7 +528,7 @@ public class MQClientAPIImpl { if (null == sendCallback && response != null) { try { - SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response); + SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); if (context != null && sendResult != null) { context.setSendResult(sendResult); context.getProducer().executeSendMessageHookAfter(context); @@ -542,7 +542,7 @@ public class MQClientAPIImpl { if (response != null) { try { - SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response); + SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); assert sendResult != null; if (context != null) { context.setSendResult(sendResult); @@ -641,7 +641,8 @@ public class MQClientAPIImpl { private SendResult processSendResponse( final String brokerName, final Message msg, - final RemotingCommand response + final RemotingCommand response, + final String addr ) throws MQBrokerException, RemotingCommandException { SendStatus sendStatus; switch (response.getCode()) { @@ -662,7 +663,7 @@ public class MQClientAPIImpl { break; } default: { - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } } @@ -741,7 +742,7 @@ public class MQClientAPIImpl { RemotingCommand response = responseFuture.getResponseCommand(); if (response != null) { try { - PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response); + PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); assert pullResult != null; pullCallback.onSuccess(pullResult); } catch (Exception e) { @@ -768,11 +769,12 @@ public class MQClientAPIImpl { ) throws RemotingException, InterruptedException, MQBrokerException { RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); assert response != null; - return this.processPullResponse(response); + return this.processPullResponse(response, addr); } private PullResult processPullResponse( - final RemotingCommand response) throws MQBrokerException, RemotingCommandException { + final RemotingCommand response, + final String addr) throws MQBrokerException, RemotingCommandException { PullStatus pullStatus = PullStatus.NO_NEW_MSG; switch (response.getCode()) { case ResponseCode.SUCCESS: @@ -789,7 +791,7 @@ public class MQClientAPIImpl { break; default: - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } PullMessageResponseHeader responseHeader = @@ -822,7 +824,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public long searchOffset(final String addr, final String topic, final int queueId, final long timestamp, @@ -847,7 +849,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public long getMaxOffset(final String addr, final String topic, final int queueId, final long timeoutMillis) @@ -871,7 +873,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public List getConsumerIdListByGroup( @@ -898,7 +900,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public long getMinOffset(final String addr, final String topic, final int queueId, final long timeoutMillis) @@ -922,7 +924,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public long getEarliestMsgStoretime(final String addr, final String topic, final int queueId, @@ -947,7 +949,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public long queryConsumerOffset( @@ -971,7 +973,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public void updateConsumerOffset( @@ -992,7 +994,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public void updateConsumerOffsetOneway( @@ -1024,7 +1026,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public void unregisterClient( @@ -1050,7 +1052,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public void endTransactionOneway( @@ -1116,7 +1118,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public Set lockBatchMQ( @@ -1138,7 +1140,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public void unlockBatchMQ( @@ -1164,7 +1166,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } } @@ -1187,7 +1189,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final long timeoutMillis) @@ -1217,7 +1219,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public ProducerConnection getProducerConnectionList(final String addr, final String producerGroup, @@ -1239,7 +1241,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public ConsumerConnection getConsumerConnectionList(final String addr, final String consumerGroup, @@ -1261,7 +1263,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public KVTable getBrokerRuntimeInfo(final String addr, final long timeoutMillis) throws RemotingConnectException, @@ -1279,7 +1281,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public void updateBrokerConfig(final String addr, final Properties properties, final long timeoutMillis) @@ -1301,7 +1303,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } } @@ -1320,7 +1322,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public ClusterInfo getBrokerClusterInfo( @@ -1364,7 +1366,7 @@ public class MQClientAPIImpl { assert response != null; switch (response.getCode()) { case ResponseCode.TOPIC_NOT_EXIST: { - if (allowTopicNotExist && !topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { + if (allowTopicNotExist) { log.warn("get Topic [{}] RouteInfoFromNameServer is not exist value", topic); } @@ -1465,10 +1467,11 @@ public class MQClientAPIImpl { throw new MQClientException(response.getCode(), response.getRemark()); } - public void deleteSubscriptionGroup(final String addr, final String groupName, final long timeoutMillis) + public void deleteSubscriptionGroup(final String addr, final String groupName, final boolean removeOffset, final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { DeleteSubscriptionGroupRequestHeader requestHeader = new DeleteSubscriptionGroupRequestHeader(); requestHeader.setGroupName(groupName); + requestHeader.setRemoveOffset(removeOffset); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, requestHeader); RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), @@ -1670,7 +1673,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public List queryConsumeTimeSpan(final String addr, final String topic, final String group, @@ -1694,7 +1697,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public TopicList getTopicsByCluster(final String cluster, final long timeoutMillis) @@ -1745,7 +1748,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public TopicList getSystemTopicList( @@ -2108,7 +2111,7 @@ public class MQClientAPIImpl { default: break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr); } public TopicConfigSerializeWrapper getAllTopicConfig(final String addr, @@ -2127,7 +2130,7 @@ public class MQClientAPIImpl { break; } - throw new MQBrokerException(response.getCode(), response.getRemark()); + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } public void updateNameServerConfig(final Properties properties, final List nameServers, long timeoutMillis) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java index fad0b4f1b584d301c7683aac3be36531627e2721..4d18a9be1b7c7fbdb870fef98a9d45075879c142 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java @@ -83,9 +83,12 @@ public class AssignedMessageQueue { return -1; } - public void updatePullOffset(MessageQueue messageQueue, long offset) { + public void updatePullOffset(MessageQueue messageQueue, long offset, ProcessQueue processQueue) { MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); if (messageQueueState != null) { + if (messageQueueState.getProcessQueue() != processQueue) { + return; + } messageQueueState.setPullOffset(offset); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java index 258e4dbf8777fd6c5d1f0aa0cb947da332c6064f..b37f8a635983573c28d42a6708e0ed1ffee91b92 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java @@ -45,6 +45,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.body.CMResult; import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -92,9 +93,9 @@ public class ConsumeMessageConcurrentlyService implements ConsumeMessageService }, this.defaultMQPushConsumer.getConsumeTimeout(), this.defaultMQPushConsumer.getConsumeTimeout(), TimeUnit.MINUTES); } - public void shutdown() { + public void shutdown(long awaitTerminateMillis) { this.scheduledExecutorService.shutdown(); - this.consumeExecutor.shutdown(); + ThreadUtils.shutdownGracefully(this.consumeExecutor, awaitTerminateMillis, TimeUnit.MILLISECONDS); this.cleanExpireMsgExecutors.shutdown(); } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java index f65567b685d723a5b65b465f0a93c363c985d450..130effad9e55f08ede275f6870ba9976f11f5f56 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.protocol.NamespaceUtil; +import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; @@ -96,10 +97,10 @@ public class ConsumeMessageOrderlyService implements ConsumeMessageService { } } - public void shutdown() { + public void shutdown(long awaitTerminateMillis) { this.stopped = true; this.scheduledExecutorService.shutdown(); - this.consumeExecutor.shutdown(); + ThreadUtils.shutdownGracefully(this.consumeExecutor, awaitTerminateMillis, TimeUnit.MILLISECONDS); if (MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) { this.unlockAllMQ(); } @@ -279,7 +280,7 @@ public class ConsumeMessageOrderlyService implements ConsumeMessageService { case SUSPEND_CURRENT_QUEUE_A_MOMENT: this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), msgs.size()); if (checkReconsumeTimes(msgs)) { - consumeRequest.getProcessQueue().makeMessageToCosumeAgain(msgs); + consumeRequest.getProcessQueue().makeMessageToConsumeAgain(msgs); this.submitConsumeRequestLater( consumeRequest.getProcessQueue(), consumeRequest.getMessageQueue(), @@ -311,7 +312,7 @@ public class ConsumeMessageOrderlyService implements ConsumeMessageService { case SUSPEND_CURRENT_QUEUE_A_MOMENT: this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), msgs.size()); if (checkReconsumeTimes(msgs)) { - consumeRequest.getProcessQueue().makeMessageToCosumeAgain(msgs); + consumeRequest.getProcessQueue().makeMessageToConsumeAgain(msgs); this.submitConsumeRequestLater( consumeRequest.getProcessQueue(), consumeRequest.getMessageQueue(), @@ -477,7 +478,7 @@ public class ConsumeMessageOrderlyService implements ConsumeMessageService { ConsumeReturnType returnType = ConsumeReturnType.SUCCESS; boolean hasException = false; try { - this.processQueue.getLockConsume().lock(); + this.processQueue.getConsumeLock().lock(); if (this.processQueue.isDropped()) { log.warn("consumeMessage, the message queue not be able to consume, because it's dropped. {}", this.messageQueue); @@ -493,7 +494,7 @@ public class ConsumeMessageOrderlyService implements ConsumeMessageService { messageQueue); hasException = true; } finally { - this.processQueue.getLockConsume().unlock(); + this.processQueue.getConsumeLock().unlock(); } if (null == status diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageService.java index 0f6f3bb38af6d22e02044bdaa738d2a8d45c2e8c..5078c97883502f198d1d2f395b42b0f4ee6262bb 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageService.java @@ -24,7 +24,7 @@ import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; public interface ConsumeMessageService { void start(); - void shutdown(); + void shutdown(long awaitTerminateMillis); void updateCorePoolSize(int corePoolSize); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index f54078fcfdc687a650444e147d5cbb59779b6fe9..4e139c44ce0c25d0eee8b01bc2f87804a036c8b9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -41,12 +41,15 @@ import org.apache.rocketmq.client.consumer.MessageQueueListener; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.TopicMessageQueueChangeListener; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore; import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.consumer.store.ReadOffsetType; import org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.ConsumeMessageContext; +import org.apache.rocketmq.client.hook.ConsumeMessageHook; import org.apache.rocketmq.client.hook.FilterMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; @@ -115,6 +118,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { */ private static final long PULL_TIME_DELAY_MILLS_WHEN_PAUSE = 1000; + private static final long PULL_TIME_DELAY_MILLS_ON_EXCEPTION = 3 * 1000; + private DefaultLitePullConsumer defaultLitePullConsumer; private final ConcurrentMap taskTable = @@ -142,6 +147,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private final MessageQueueLock messageQueueLock = new MessageQueueLock(); + private final ArrayList consumeMessageHookList = new ArrayList<>(); + public DefaultLitePullConsumerImpl(final DefaultLitePullConsumer defaultLitePullConsumer, final RPCHook rpcHook) { this.defaultLitePullConsumer = defaultLitePullConsumer; this.rpcHook = rpcHook; @@ -158,6 +165,35 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { this.pullTimeDelayMillsWhenException = defaultLitePullConsumer.getPullTimeDelayMillsWhenException(); } + public void registerConsumeMessageHook(final ConsumeMessageHook hook) { + this.consumeMessageHookList.add(hook); + log.info("register consumeMessageHook Hook, {}", hook.hookName()); + } + + public void executeHookBefore(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageBefore(context); + } catch (Throwable e) { + log.error("consumeMessageHook {} executeHookBefore exception", hook.hookName(), e); + } + } + } + } + + public void executeHookAfter(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageAfter(context); + } catch (Throwable e) { + log.error("consumeMessageHook {} executeHookAfter exception", hook.hookName(), e); + } + } + } + } + private void checkServiceState() { if (this.serviceState != ServiceState.RUNNING) throw new IllegalStateException(NOT_RUNNING_EXCEPTION_MESSAGE); @@ -229,6 +265,10 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } + public synchronized boolean isRunning() { + return this.serviceState == ServiceState.RUNNING; + } + public synchronized void start() throws MQClientException { switch (this.serviceState) { case CREATE_JUST: @@ -428,8 +468,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { throw new IllegalArgumentException("Topic can not be null or empty."); } setSubscriptionType(SubscriptionType.SUBSCRIBE); - SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(defaultLitePullConsumer.getConsumerGroup(), - topic, subExpression); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, subExpression); this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); this.defaultLitePullConsumer.setMessageQueueListener(new MessageQueueListenerImpl()); assignedMessageQueue.setRebalanceImpl(this.rebalanceImpl); @@ -609,9 +648,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private void updatePullOffset(MessageQueue messageQueue, long nextPullOffset) { + private void updatePullOffset(MessageQueue messageQueue, long nextPullOffset, ProcessQueue processQueue) { if (assignedMessageQueue.getSeekOffset(messageQueue) == -1) { - assignedMessageQueue.updatePullOffset(messageQueue, nextPullOffset); + assignedMessageQueue.updatePullOffset(messageQueue, nextPullOffset, processQueue); } } @@ -623,9 +662,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private long fetchConsumeOffset(MessageQueue messageQueue) { + private long fetchConsumeOffset(MessageQueue messageQueue) throws MQClientException { checkServiceState(); - long offset = this.rebalanceImpl.computePullFromWhere(messageQueue); + long offset = this.rebalanceImpl.computePullFromWhereWithException(messageQueue); return offset; } @@ -649,7 +688,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private long nextPullOffset(MessageQueue messageQueue) { + private long nextPullOffset(MessageQueue messageQueue) throws MQClientException { long offset = -1; long seekOffset = assignedMessageQueue.getSeekOffset(messageQueue); if (seekOffset != -1) { @@ -691,7 +730,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); - if (processQueue == null && processQueue.isDropped()) { + if (null == processQueue || processQueue.isDropped()) { log.info("The message queue not be able to poll, because it's dropped. group={}, messageQueue={}", defaultLitePullConsumer.getConsumerGroup(), this.messageQueue); return; } @@ -736,7 +775,18 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return; } - long offset = nextPullOffset(messageQueue); + long offset = 0L; + try { + offset = nextPullOffset(messageQueue); + } catch (MQClientException e) { + log.error("Failed to get next pull offset", e); + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_ON_EXCEPTION, TimeUnit.MILLISECONDS); + return; + } + + if (this.isCancelled() || processQueue.isDropped()) { + return; + } long pullDelayTimeMills = 0; try { SubscriptionData subscriptionData; @@ -745,12 +795,13 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { subscriptionData = rebalanceImpl.getSubscriptionInner().get(topic); } else { String topic = this.messageQueue.getTopic(); - subscriptionData = FilterAPI.buildSubscriptionData(defaultLitePullConsumer.getConsumerGroup(), - topic, SubscriptionData.SUB_ALL); + subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL); } PullResult pullResult = pull(messageQueue, subscriptionData, offset, defaultLitePullConsumer.getPullBatchSize()); - + if (this.isCancelled() || processQueue.isDropped()) { + return; + } switch (pullResult.getPullStatus()) { case FOUND: final Object objLock = messageQueueLock.fetchLockObject(messageQueue); @@ -767,7 +818,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { default: break; } - updatePullOffset(messageQueue, pullResult.getNextBeginOffset()); + updatePullOffset(messageQueue, pullResult.getNextBeginOffset(), processQueue); } catch (Throwable e) { pullDelayTimeMills = pullTimeDelayMillsWhenException; log.error("An error occurred in pull message process.", e); @@ -841,6 +892,18 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { null ); this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); + if (!this.consumeMessageHookList.isEmpty()) { + ConsumeMessageContext consumeMessageContext = new ConsumeMessageContext(); + consumeMessageContext.setNamespace(defaultLitePullConsumer.getNamespace()); + consumeMessageContext.setConsumerGroup(this.groupName()); + consumeMessageContext.setMq(mq); + consumeMessageContext.setMsgList(pullResult.getMsgFoundList()); + consumeMessageContext.setSuccess(false); + this.executeHookBefore(consumeMessageContext); + consumeMessageContext.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString()); + consumeMessageContext.setSuccess(true); + this.executeHookAfter(consumeMessageContext); + } return pullResult; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index afd72a08002c6c9124feb6260f011e9810407842..eed5fa43f04fff819345def276320a442ee3f9fc 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -205,8 +205,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { } try { - return FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(), - mq.getTopic(), subExpression); + return FilterAPI.buildSubscriptionData(mq.getTopic(), subExpression); } catch (Exception e) { throw new MQClientException("parse subscription error", e); } @@ -301,8 +300,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { public void subscriptionAutomatically(final String topic) { if (!this.rebalanceImpl.getSubscriptionInner().containsKey(topic)) { try { - SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(), - topic, SubscriptionData.SUB_ALL); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL); this.rebalanceImpl.subscriptionInner.putIfAbsent(topic, subscriptionData); } catch (Exception ignore) { } @@ -365,7 +363,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { for (String t : topics) { SubscriptionData ms = null; try { - ms = FilterAPI.buildSubscriptionData(this.groupName(), t, SubscriptionData.SUB_ALL); + ms = FilterAPI.buildSubscriptionData(t, SubscriptionData.SUB_ALL); } catch (Exception e) { log.error("parse subscription error", e); } @@ -742,8 +740,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { Set registerTopics = this.defaultMQPullConsumer.getRegisterTopics(); if (registerTopics != null) { for (final String topic : registerTopics) { - SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(), - topic, SubscriptionData.SUB_ALL); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL); this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 25a81a0e755e60e968ae281d4390cff5b77edcc7..bb0b7f10436ad7ef0844bad9ec09849c72b5940c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -269,8 +269,15 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { } } else { if (processQueue.isLocked()) { - if (!pullRequest.isLockedFirst()) { - final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue()); + if (!pullRequest.isPreviouslyLocked()) { + long offset = -1L; + try { + offset = this.rebalanceImpl.computePullFromWhereWithException(pullRequest.getMessageQueue()); + } catch (MQClientException e) { + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); + log.error("Failed to compute pull offset, pullResult: {}", pullRequest, e); + return; + } boolean brokerBusy = offset < pullRequest.getNextOffset(); log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}", pullRequest, offset, brokerBusy); @@ -279,7 +286,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { pullRequest, offset); } - pullRequest.setLockedFirst(true); + pullRequest.setPreviouslyLocked(true); pullRequest.setNextOffset(offset); } } else { @@ -348,12 +355,6 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { break; case NO_NEW_MSG: - pullRequest.setNextOffset(pullResult.getNextBeginOffset()); - - DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); - - DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); - break; case NO_MATCHED_MSG: pullRequest.setNextOffset(pullResult.getNextBeginOffset()); @@ -546,12 +547,16 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { } } - public synchronized void shutdown() { + public void shutdown() { + shutdown(0); + } + + public synchronized void shutdown(long awaitTerminateMillis) { switch (this.serviceState) { case CREATE_JUST: break; case RUNNING: - this.consumeMessageService.shutdown(); + this.consumeMessageService.shutdown(awaitTerminateMillis); this.persistConsumerOffset(); this.mQClientFactory.unregisterConsumer(this.defaultMQPushConsumer.getConsumerGroup()); this.mQClientFactory.shutdown(); @@ -625,7 +630,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPushConsumer.getConsumerGroup(), this); if (!registerOK) { this.serviceState = ServiceState.CREATE_JUST; - this.consumeMessageService.shutdown(); + this.consumeMessageService.shutdown(defaultMQPushConsumer.getAwaitTerminationMillisWhenShutdown()); throw new MQClientException("The consumer group[" + this.defaultMQPushConsumer.getConsumerGroup() + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), null); @@ -829,8 +834,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { for (final Map.Entry entry : sub.entrySet()) { final String topic = entry.getKey(); final String subString = entry.getValue(); - SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), - topic, subString); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, subString); this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); } } @@ -844,8 +848,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { break; case CLUSTERING: final String retryTopic = MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()); - SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), - retryTopic, SubscriptionData.SUB_ALL); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(retryTopic, SubscriptionData.SUB_ALL); this.rebalanceImpl.getSubscriptionInner().put(retryTopic, subscriptionData); break; default: @@ -876,8 +879,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { public void subscribe(String topic, String subExpression) throws MQClientException { try { - SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), - topic, subExpression); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, subExpression); this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); if (this.mQClientFactory != null) { this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); @@ -889,8 +891,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { public void subscribe(String topic, String fullClassName, String filterClassSource) throws MQClientException { try { - SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), - topic, "*"); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(topic, "*"); subscriptionData.setSubString(fullClassName); subscriptionData.setClassFilterMode(true); subscriptionData.setFilterClassSource(filterClassSource); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java index f659bd3f83100c5a1837f4ecadc7bbfdee9f63aa..21798d8c6ec03d6bb82d5cf32b5c84327be4dc42 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java @@ -44,11 +44,11 @@ public class ProcessQueue { public final static long REBALANCE_LOCK_INTERVAL = Long.parseLong(System.getProperty("rocketmq.client.rebalance.lockInterval", "20000")); private final static long PULL_MAX_IDLE_TIME = Long.parseLong(System.getProperty("rocketmq.client.pull.pullMaxIdleTime", "120000")); private final InternalLogger log = ClientLogger.getLog(); - private final ReadWriteLock lockTreeMap = new ReentrantReadWriteLock(); + private final ReadWriteLock treeMapLock = new ReentrantReadWriteLock(); private final TreeMap msgTreeMap = new TreeMap(); private final AtomicLong msgCount = new AtomicLong(); private final AtomicLong msgSize = new AtomicLong(); - private final Lock lockConsume = new ReentrantLock(); + private final Lock consumeLock = new ReentrantLock(); /** * A subset of msgTreeMap, will only be used when orderly consume */ @@ -83,7 +83,7 @@ public class ProcessQueue { for (int i = 0; i < loop; i++) { MessageExt msg = null; try { - this.lockTreeMap.readLock().lockInterruptibly(); + this.treeMapLock.readLock().lockInterruptibly(); try { if (!msgTreeMap.isEmpty() && System.currentTimeMillis() - Long.parseLong(MessageAccessor.getConsumeStartTimeStamp(msgTreeMap.firstEntry().getValue())) > pushConsumer.getConsumeTimeout() * 60 * 1000) { msg = msgTreeMap.firstEntry().getValue(); @@ -92,7 +92,7 @@ public class ProcessQueue { break; } } finally { - this.lockTreeMap.readLock().unlock(); + this.treeMapLock.readLock().unlock(); } } catch (InterruptedException e) { log.error("getExpiredMsg exception", e); @@ -103,7 +103,7 @@ public class ProcessQueue { pushConsumer.sendMessageBack(msg, 3); log.info("send expire msg back. topic={}, msgId={}, storeHost={}, queueId={}, queueOffset={}", msg.getTopic(), msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset()); try { - this.lockTreeMap.writeLock().lockInterruptibly(); + this.treeMapLock.writeLock().lockInterruptibly(); try { if (!msgTreeMap.isEmpty() && msg.getQueueOffset() == msgTreeMap.firstKey()) { try { @@ -113,7 +113,7 @@ public class ProcessQueue { } } } finally { - this.lockTreeMap.writeLock().unlock(); + this.treeMapLock.writeLock().unlock(); } } catch (InterruptedException e) { log.error("getExpiredMsg exception", e); @@ -127,7 +127,7 @@ public class ProcessQueue { public boolean putMessage(final List msgs) { boolean dispatchToConsume = false; try { - this.lockTreeMap.writeLock().lockInterruptibly(); + this.treeMapLock.writeLock().lockInterruptibly(); try { int validMsgCnt = 0; for (MessageExt msg : msgs) { @@ -156,7 +156,7 @@ public class ProcessQueue { } } } finally { - this.lockTreeMap.writeLock().unlock(); + this.treeMapLock.writeLock().unlock(); } } catch (InterruptedException e) { log.error("putMessage exception", e); @@ -167,13 +167,13 @@ public class ProcessQueue { public long getMaxSpan() { try { - this.lockTreeMap.readLock().lockInterruptibly(); + this.treeMapLock.readLock().lockInterruptibly(); try { if (!this.msgTreeMap.isEmpty()) { return this.msgTreeMap.lastKey() - this.msgTreeMap.firstKey(); } } finally { - this.lockTreeMap.readLock().unlock(); + this.treeMapLock.readLock().unlock(); } } catch (InterruptedException e) { log.error("getMaxSpan exception", e); @@ -186,7 +186,7 @@ public class ProcessQueue { long result = -1; final long now = System.currentTimeMillis(); try { - this.lockTreeMap.writeLock().lockInterruptibly(); + this.treeMapLock.writeLock().lockInterruptibly(); this.lastConsumeTimestamp = now; try { if (!msgTreeMap.isEmpty()) { @@ -206,7 +206,7 @@ public class ProcessQueue { } } } finally { - this.lockTreeMap.writeLock().unlock(); + this.treeMapLock.writeLock().unlock(); } } catch (Throwable t) { log.error("removeMessage exception", t); @@ -245,12 +245,12 @@ public class ProcessQueue { public void rollback() { try { - this.lockTreeMap.writeLock().lockInterruptibly(); + this.treeMapLock.writeLock().lockInterruptibly(); try { this.msgTreeMap.putAll(this.consumingMsgOrderlyTreeMap); this.consumingMsgOrderlyTreeMap.clear(); } finally { - this.lockTreeMap.writeLock().unlock(); + this.treeMapLock.writeLock().unlock(); } } catch (InterruptedException e) { log.error("rollback exception", e); @@ -259,7 +259,7 @@ public class ProcessQueue { public long commit() { try { - this.lockTreeMap.writeLock().lockInterruptibly(); + this.treeMapLock.writeLock().lockInterruptibly(); try { Long offset = this.consumingMsgOrderlyTreeMap.lastKey(); msgCount.addAndGet(0 - this.consumingMsgOrderlyTreeMap.size()); @@ -271,7 +271,7 @@ public class ProcessQueue { return offset + 1; } } finally { - this.lockTreeMap.writeLock().unlock(); + this.treeMapLock.writeLock().unlock(); } } catch (InterruptedException e) { log.error("commit exception", e); @@ -280,16 +280,16 @@ public class ProcessQueue { return -1; } - public void makeMessageToCosumeAgain(List msgs) { + public void makeMessageToConsumeAgain(List msgs) { try { - this.lockTreeMap.writeLock().lockInterruptibly(); + this.treeMapLock.writeLock().lockInterruptibly(); try { for (MessageExt msg : msgs) { this.consumingMsgOrderlyTreeMap.remove(msg.getQueueOffset()); this.msgTreeMap.put(msg.getQueueOffset(), msg); } } finally { - this.lockTreeMap.writeLock().unlock(); + this.treeMapLock.writeLock().unlock(); } } catch (InterruptedException e) { log.error("makeMessageToCosumeAgain exception", e); @@ -300,7 +300,7 @@ public class ProcessQueue { List result = new ArrayList(batchSize); final long now = System.currentTimeMillis(); try { - this.lockTreeMap.writeLock().lockInterruptibly(); + this.treeMapLock.writeLock().lockInterruptibly(); this.lastConsumeTimestamp = now; try { if (!this.msgTreeMap.isEmpty()) { @@ -319,7 +319,7 @@ public class ProcessQueue { consuming = false; } } finally { - this.lockTreeMap.writeLock().unlock(); + this.treeMapLock.writeLock().unlock(); } } catch (InterruptedException e) { log.error("take Messages exception", e); @@ -330,11 +330,11 @@ public class ProcessQueue { public boolean hasTempMessage() { try { - this.lockTreeMap.readLock().lockInterruptibly(); + this.treeMapLock.readLock().lockInterruptibly(); try { return !this.msgTreeMap.isEmpty(); } finally { - this.lockTreeMap.readLock().unlock(); + this.treeMapLock.readLock().unlock(); } } catch (InterruptedException e) { } @@ -344,7 +344,7 @@ public class ProcessQueue { public void clear() { try { - this.lockTreeMap.writeLock().lockInterruptibly(); + this.treeMapLock.writeLock().lockInterruptibly(); try { this.msgTreeMap.clear(); this.consumingMsgOrderlyTreeMap.clear(); @@ -352,7 +352,7 @@ public class ProcessQueue { this.msgSize.set(0); this.queueOffsetMax = 0L; } finally { - this.lockTreeMap.writeLock().unlock(); + this.treeMapLock.writeLock().unlock(); } } catch (InterruptedException e) { log.error("rollback exception", e); @@ -367,8 +367,8 @@ public class ProcessQueue { this.lastLockTimestamp = lastLockTimestamp; } - public Lock getLockConsume() { - return lockConsume; + public Lock getConsumeLock() { + return consumeLock; } public long getLastPullTimestamp() { @@ -397,7 +397,7 @@ public class ProcessQueue { public void fillProcessQueueInfo(final ProcessQueueInfo info) { try { - this.lockTreeMap.readLock().lockInterruptibly(); + this.treeMapLock.readLock().lockInterruptibly(); if (!this.msgTreeMap.isEmpty()) { info.setCachedMsgMinOffset(this.msgTreeMap.firstKey()); @@ -421,7 +421,7 @@ public class ProcessQueue { info.setLastConsumeTimestamp(this.lastConsumeTimestamp); } catch (Exception e) { } finally { - this.lockTreeMap.readLock().unlock(); + this.treeMapLock.readLock().unlock(); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java index 231ac0c68f3e26d099de8318c62741d320e7cb57..cc42a9e830ee9dde11869423dd879cefdd9d967a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java @@ -194,7 +194,7 @@ public class PullAPIWrapper { String brokerAddr = findBrokerResult.getBrokerAddr(); if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) { - brokerAddr = computPullFromWhichFilterServer(mq.getTopic(), brokerAddr); + brokerAddr = computePullFromWhichFilterServer(mq.getTopic(), brokerAddr); } PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage( @@ -223,7 +223,7 @@ public class PullAPIWrapper { return MixAll.MASTER_ID; } - private String computPullFromWhichFilterServer(final String topic, final String brokerAddr) + private String computePullFromWhichFilterServer(final String topic, final String brokerAddr) throws MQClientException { ConcurrentMap topicRouteTable = this.mQClientFactory.getTopicRouteTable(); if (topicRouteTable != null) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java index 10aded076252b7513b5eb42c67ebb28136f42930..bf03ec38c3a68d9a789504f3291b9b66d4c833cf 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java @@ -23,14 +23,14 @@ public class PullRequest { private MessageQueue messageQueue; private ProcessQueue processQueue; private long nextOffset; - private boolean lockedFirst = false; + private boolean previouslyLocked = false; - public boolean isLockedFirst() { - return lockedFirst; + public boolean isPreviouslyLocked() { + return previouslyLocked; } - public void setLockedFirst(boolean lockedFirst) { - this.lockedFirst = lockedFirst; + public void setPreviouslyLocked(boolean previouslyLocked) { + this.previouslyLocked = previouslyLocked; } public String getConsumerGroup() { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java index b8972a92e8fdb3402b92c7c1b1a8015cb38a127b..833d465a4703184be6e9a17c20682fdd5d81b240 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; @@ -373,7 +374,15 @@ public abstract class RebalanceImpl { this.removeDirtyOffset(mq); ProcessQueue pq = new ProcessQueue(); - long nextOffset = this.computePullFromWhere(mq); + + long nextOffset = -1L; + try { + nextOffset = this.computePullFromWhereWithException(mq); + } catch (MQClientException e) { + log.info("doRebalance, {}, compute offset failed, {}", consumerGroup, mq); + continue; + } + if (nextOffset >= 0) { ProcessQueue pre = this.processQueueTable.putIfAbsent(mq, pq); if (pre != null) { @@ -408,8 +417,17 @@ public abstract class RebalanceImpl { public abstract void removeDirtyOffset(final MessageQueue mq); + /** + * When the network is unstable, using this interface may return wrong offset. + * It is recommended to use computePullFromWhereWithException instead. + * @param mq + * @return offset + */ + @Deprecated public abstract long computePullFromWhere(final MessageQueue mq); + public abstract long computePullFromWhereWithException(final MessageQueue mq) throws MQClientException; + public abstract void dispatchPullRequest(final List pullRequestList); public void removeProcessQueue(final MessageQueue mq) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java index 9d1ea7492eea2ffc662defea11f4fe5d1bf84f38..286c684e43a52dc3759c0f3b9393e0ee75851ef4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java @@ -74,8 +74,20 @@ public class RebalanceLitePullImpl extends RebalanceImpl { this.litePullConsumerImpl.getOffsetStore().removeOffset(mq); } + @Deprecated @Override public long computePullFromWhere(MessageQueue mq) { + long result = -1L; + try { + result = computePullFromWhereWithException(mq); + } catch (MQClientException e) { + log.warn("Compute consume offset exception, mq={}", mq); + } + return result; + } + + @Override + public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException { ConsumeFromWhere consumeFromWhere = litePullConsumerImpl.getDefaultLitePullConsumer().getConsumeFromWhere(); long result = -1; switch (consumeFromWhere) { @@ -118,7 +130,8 @@ public class RebalanceLitePullImpl extends RebalanceImpl { try { result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } else { try { @@ -126,7 +139,8 @@ public class RebalanceLitePullImpl extends RebalanceImpl { UtilAll.YYYYMMDDHHMMSS).getTime(); result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } } else { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java index 9dd408c1d140ea4f7ce4aa31df5e5181c997b753..6df7eb7f00cb3141bda3cef444f6a88c7abeeadd 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Set; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; import org.apache.rocketmq.client.consumer.MessageQueueListener; +import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; @@ -68,11 +69,17 @@ public class RebalancePullImpl extends RebalanceImpl { this.defaultMQPullConsumerImpl.getOffsetStore().removeOffset(mq); } + @Deprecated @Override public long computePullFromWhere(MessageQueue mq) { return 0; } + @Override + public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException { + return 0; + } + @Override public void dispatchPullRequest(List pullRequestList) { } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java index e5166f35b59c003d4512f6f2e64965713ea8d051..666b696ffa89a631243e2e6533cfa82a52ff383f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java @@ -88,11 +88,11 @@ public class RebalancePushImpl extends RebalanceImpl { if (this.defaultMQPushConsumerImpl.isConsumeOrderly() && MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) { try { - if (pq.getLockConsume().tryLock(1000, TimeUnit.MILLISECONDS)) { + if (pq.getConsumeLock().tryLock(1000, TimeUnit.MILLISECONDS)) { try { return this.unlockDelay(mq, pq); } finally { - pq.getLockConsume().unlock(); + pq.getConsumeLock().unlock(); } } else { log.warn("[WRONG]mq is consuming, so can not unlock it, {}. maybe hanged for a while, {}", @@ -137,8 +137,20 @@ public class RebalancePushImpl extends RebalanceImpl { this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq); } + @Deprecated @Override public long computePullFromWhere(MessageQueue mq) { + long result = -1L; + try { + result = computePullFromWhereWithException(mq); + } catch (MQClientException e) { + log.warn("Compute consume offset exception, mq={}", mq); + } + return result; + } + + @Override + public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException { long result = -1; final ConsumeFromWhere consumeFromWhere = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumeFromWhere(); final OffsetStore offsetStore = this.defaultMQPushConsumerImpl.getOffsetStore(); @@ -159,7 +171,8 @@ public class RebalancePushImpl extends RebalanceImpl { try { result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } } else { @@ -187,7 +200,8 @@ public class RebalancePushImpl extends RebalanceImpl { try { result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } else { try { @@ -195,7 +209,8 @@ public class RebalancePushImpl extends RebalanceImpl { UtilAll.YYYYMMDDHHMMSS).getTime(); result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } } else { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 886203fe7d405ec9dede6ecbf6165cedaed72d13..81e6d8468ebf1d303330cd4ff21b91f79be58b8a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -475,7 +475,7 @@ public class MQClientInstance { this.lockHeartbeat.unlock(); } } else { - log.warn("lock heartBeat, but failed."); + log.warn("lock heartBeat, but failed. [{}]", this.clientId); } } @@ -532,7 +532,7 @@ public class MQClientInstance { final boolean producerEmpty = heartbeatData.getProducerDataSet().isEmpty(); final boolean consumerEmpty = heartbeatData.getConsumerDataSet().isEmpty(); if (producerEmpty && consumerEmpty) { - log.warn("sending heartbeat, but no consumer and no producer"); + log.warn("sending heartbeat, but no consumer and no producer. [{}]", this.clientId); return; } @@ -668,10 +668,10 @@ public class MQClientInstance { return true; } } else { - log.warn("updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}", topic); + log.warn("updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}. [{}]", topic, this.clientId); } } catch (MQClientException e) { - if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) && !topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) { + if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { log.warn("updateTopicRouteInfoFromNameServer Exception", e); } } catch (RemotingException e) { @@ -681,7 +681,7 @@ public class MQClientInstance { this.lockNamesrv.unlock(); } } else { - log.warn("updateTopicRouteInfoFromNameServer tryLock timeout {}ms", LOCK_TIMEOUT_MILLIS); + log.warn("updateTopicRouteInfoFromNameServer tryLock timeout {}ms. [{}]", LOCK_TIMEOUT_MILLIS, this.clientId); } } catch (InterruptedException e) { log.warn("updateTopicRouteInfoFromNameServer Exception", e); @@ -893,7 +893,7 @@ public class MQClientInstance { this.lockHeartbeat.unlock(); } } else { - log.warn("lock heartBeat, but failed."); + log.warn("lock heartBeat, but failed. [{}]", this.clientId); } } catch (InterruptedException e) { log.warn("unregisterClientWithLock exception", e); @@ -1043,6 +1043,11 @@ public class MQClientInstance { slave = brokerId != MixAll.MASTER_ID; found = brokerAddr != null; + if (!found && slave) { + brokerAddr = map.get(brokerId + 1); + found = brokerAddr != null; + } + if (!found && !onlyThisBroker) { Entry entry = map.entrySet().iterator().next(); brokerAddr = entry.getValue(); @@ -1100,7 +1105,7 @@ public class MQClientInstance { return null; } - public void resetOffset(String topic, String group, Map offsetTable) { + public synchronized void resetOffset(String topic, String group, Map offsetTable) { DefaultMQPushConsumerImpl consumer = null; try { MQConsumerInner impl = this.consumerTable.get(group); @@ -1209,6 +1214,9 @@ public class MQClientInstance { public ConsumerRunningInfo consumerRunningInfo(final String consumerGroup) { MQConsumerInner mqConsumerInner = this.consumerTable.get(consumerGroup); + if (mqConsumerInner == null) { + return null; + } ConsumerRunningInfo consumerRunningInfo = mqConsumerInner.consumerRunningInfo(); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index bff1457835fba85421573039410b2c79aeae6d8e..fac3ed3561f2003da0d3be1855b7995af62bd8f6 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -44,6 +44,8 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.hook.CheckForbiddenContext; import org.apache.rocketmq.client.hook.CheckForbiddenHook; +import org.apache.rocketmq.client.hook.EndTransactionContext; +import org.apache.rocketmq.client.hook.EndTransactionHook; import org.apache.rocketmq.client.hook.SendMessageContext; import org.apache.rocketmq.client.hook.SendMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; @@ -101,6 +103,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { private final ConcurrentMap topicPublishInfoTable = new ConcurrentHashMap(); private final ArrayList sendMessageHookList = new ArrayList(); + private final ArrayList endTransactionHookList = new ArrayList(); private final RPCHook rpcHook; private final BlockingQueue asyncSenderThreadPoolQueue; private final ExecutorService defaultAsyncSenderExecutor; @@ -171,6 +174,11 @@ public class DefaultMQProducerImpl implements MQProducerInner { log.info("register sendMessage Hook, {}", hook.hookName()); } + public void registerEndTransactionHook(final EndTransactionHook hook) { + this.endTransactionHookList.add(hook); + log.info("register endTransaction Hook, {}", hook.hookName()); + } + public void start() throws MQClientException { this.start(true); } @@ -386,6 +394,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { if (exception != null) { remark = "checkLocalTransactionState Exception: " + RemotingHelper.exceptionSimpleDesc(exception); } + doExecuteEndTransactionHook(msg, uniqueKey, brokerAddr, localTransactionState, true); try { DefaultMQProducerImpl.this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, thisHeader, remark, @@ -421,6 +430,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { this.makeSureStateOK(); Validators.checkTopic(newTopic); + Validators.isSystemTopic(newTopic); this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); } @@ -966,6 +976,36 @@ public class DefaultMQProducerImpl implements MQProducerInner { } } + public boolean hasEndTransactionHook() { + return !this.endTransactionHookList.isEmpty(); + } + + public void executeEndTransactionHook(final EndTransactionContext context) { + if (!this.endTransactionHookList.isEmpty()) { + for (EndTransactionHook hook : this.endTransactionHookList) { + try { + hook.endTransaction(context); + } catch (Throwable e) { + log.warn("failed to executeEndTransactionHook", e); + } + } + } + } + + public void doExecuteEndTransactionHook(Message msg, String msgId, String brokerAddr, LocalTransactionState state, + boolean fromTransactionCheck) { + if (hasEndTransactionHook()) { + EndTransactionContext context = new EndTransactionContext(); + context.setProducerGroup(defaultMQProducer.getProducerGroup()); + context.setBrokerAddr(brokerAddr); + context.setMessage(msg); + context.setMsgId(msgId); + context.setTransactionId(msg.getTransactionId()); + context.setTransactionState(state); + context.setFromTransactionCheck(fromTransactionCheck); + executeEndTransactionHook(context); + } + } /** * DEFAULT ONEWAY ------------------------------------------------------- */ @@ -1114,7 +1154,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { mq = mQClientFactory.getClientConfig().queueWithNamespace(selector.select(messageQueueList, userMessage, arg)); } catch (Throwable e) { - throw new MQClientException("select message queue throwed exception.", e); + throw new MQClientException("select message queue threw exception.", e); } long costTime = System.currentTimeMillis() - beginStartTime; @@ -1265,7 +1305,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } try { - this.endTransaction(sendResult, localTransactionState, localException); + this.endTransaction(msg, sendResult, localTransactionState, localException); } catch (Exception e) { log.warn("local transaction execute " + localTransactionState + ", but end broker transaction failed", e); } @@ -1289,6 +1329,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } public void endTransaction( + final Message msg, final SendResult sendResult, final LocalTransactionState localTransactionState, final Throwable localException) throws RemotingException, MQBrokerException, InterruptedException, UnknownHostException { @@ -1317,6 +1358,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { break; } + doExecuteEndTransactionHook(msg, sendResult.getMsgId(), brokerAddr, localTransactionState, false); requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); requestHeader.setTranStateTableOffset(sendResult.getQueueOffset()); requestHeader.setMsgId(sendResult.getMsgId()); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java index deb02cff285b42c241e4c064ad376d47ebdecfa8..2f8337edefdd099805ad6ae9dec78faf17d6887e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java @@ -70,9 +70,9 @@ public class TopicPublishInfo { if (lastBrokerName == null) { return selectOneMessageQueue(); } else { - int index = this.sendWhichQueue.getAndIncrement(); for (int i = 0; i < this.messageQueueList.size(); i++) { - int pos = Math.abs(index++) % this.messageQueueList.size(); + int index = this.sendWhichQueue.incrementAndGet(); + int pos = Math.abs(index) % this.messageQueueList.size(); if (pos < 0) pos = 0; MessageQueue mq = this.messageQueueList.get(pos); @@ -85,7 +85,7 @@ public class TopicPublishInfo { } public MessageQueue selectOneMessageQueue() { - int index = this.sendWhichQueue.getAndIncrement(); + int index = this.sendWhichQueue.incrementAndGet(); int pos = Math.abs(index) % this.messageQueueList.size(); if (pos < 0) pos = 0; diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java index 72d43476f8371920e78153886b5ec4cff100f196..827d97265f51319a2fcc523e2b80f9062a3ef093 100644 --- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java @@ -80,7 +80,7 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance if (half <= 0) { return tmpList.get(0).getName(); } else { - final int i = this.whichItemWorst.getAndIncrement() % half; + final int i = this.whichItemWorst.incrementAndGet() % half; return tmpList.get(i).getName(); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java index 7854fcb7d10905602e27ac06d6cb9326e82e2b86..ea3d07e6d0f5028dcc35df88b6729754623367c2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java @@ -58,16 +58,14 @@ public class MQFaultStrategy { public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { if (this.sendLatencyFaultEnable) { try { - int index = tpInfo.getSendWhichQueue().getAndIncrement(); + int index = tpInfo.getSendWhichQueue().incrementAndGet(); for (int i = 0; i < tpInfo.getMessageQueueList().size(); i++) { int pos = Math.abs(index++) % tpInfo.getMessageQueueList().size(); if (pos < 0) pos = 0; MessageQueue mq = tpInfo.getMessageQueueList().get(pos); - if (latencyFaultTolerance.isAvailable(mq.getBrokerName())) { - if (null == lastBrokerName || mq.getBrokerName().equals(lastBrokerName)) - return mq; - } + if (latencyFaultTolerance.isAvailable(mq.getBrokerName())) + return mq; } final String notBestBroker = latencyFaultTolerance.pickOneAtLeast(); @@ -76,7 +74,7 @@ public class MQFaultStrategy { final MessageQueue mq = tpInfo.selectOneMessageQueue(); if (notBestBroker != null) { mq.setBrokerName(notBestBroker); - mq.setQueueId(tpInfo.getSendWhichQueue().getAndIncrement() % writeQueueNums); + mq.setQueueId(tpInfo.getSendWhichQueue().incrementAndGet() % writeQueueNums); } return mq; } else { diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index 045734103f571e798508e4dfab3eaac738586c5e..1c4a9315a8cd95c6c2aa0b1fd62c25091fa4213c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -29,6 +29,7 @@ import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; import org.apache.rocketmq.client.trace.TraceDispatcher; +import org.apache.rocketmq.client.trace.hook.EndTransactionTraceHookImpl; import org.apache.rocketmq.client.trace.hook.SendMessageTraceHookImpl; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.Message; @@ -38,6 +39,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -74,7 +76,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { /** * Just for testing or demo program */ - private String createTopicKey = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC; + private String createTopicKey = TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC; /** * Number of queues to create per default topic. @@ -166,6 +168,8 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { traceDispatcher = dispatcher; this.defaultMQProducerImpl.registerSendMessageHook( new SendMessageTraceHookImpl(traceDispatcher)); + this.defaultMQProducerImpl.registerEndTransactionHook( + new EndTransactionTraceHookImpl(traceDispatcher)); } catch (Throwable e) { log.error("system mqtrace hook init failed ,maybe can't send msg trace data"); } @@ -251,6 +255,8 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { traceDispatcher = dispatcher; this.getDefaultMQProducerImpl().registerSendMessageHook( new SendMessageTraceHookImpl(traceDispatcher)); + this.defaultMQProducerImpl.registerEndTransactionHook( + new EndTransactionTraceHookImpl(traceDispatcher)); } catch (Throwable e) { log.error("system mqtrace hook init failed ,maybe can't send msg trace data"); } @@ -916,6 +922,29 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { return this.defaultMQProducerImpl.send(batch(msgs), messageQueue, timeout); } + @Override + public void send(Collection msgs, SendCallback sendCallback) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + this.defaultMQProducerImpl.send(batch(msgs), sendCallback); + } + + @Override + public void send(Collection msgs, SendCallback sendCallback, + long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + this.defaultMQProducerImpl.send(batch(msgs), sendCallback, timeout); + } + + @Override + public void send(Collection msgs, MessageQueue mq, + SendCallback sendCallback) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + this.defaultMQProducerImpl.send(batch(msgs), queueWithNamespace(mq), sendCallback); + } + + @Override + public void send(Collection msgs, MessageQueue mq, + SendCallback sendCallback, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + this.defaultMQProducerImpl.send(batch(msgs), queueWithNamespace(mq), sendCallback, timeout); + } + /** * Sets an Executor to be used for executing callback methods. If the Executor is not set, {@link * NettyRemotingClient#publicExecutor} will be used. diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java index c6cf4c93596578e958de17815fa9604e2c09b7fe..f70ddb283da9e32093f37fcc58e23c6c5afa2aff 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java @@ -99,7 +99,19 @@ public interface MQProducer extends MQAdmin { SendResult send(final Collection msgs, final MessageQueue mq, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException; - + + void send(final Collection msgs, final SendCallback sendCallback) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException; + + void send(final Collection msgs, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, + MQBrokerException, InterruptedException; + + void send(final Collection msgs, final MessageQueue mq, final SendCallback sendCallback) throws MQClientException, RemotingException, + MQBrokerException, InterruptedException; + + void send(final Collection msgs, final MessageQueue mq, final SendCallback sendCallback, final long timeout) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException; + //for rpc Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException; diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java index 63b512df7d5374857624d81892acdf2c61252814..4eb758df401f2eea5224c75b85c8b0972339b059 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/TransactionMQProducer.java @@ -51,6 +51,10 @@ public class TransactionMQProducer extends DefaultMQProducer { super(namespace, producerGroup, rpcHook); } + public TransactionMQProducer(final String namespace, final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace, final String customizedTraceTopic) { + super(namespace, producerGroup, rpcHook, enableMsgTrace, customizedTraceTopic); + } + @Override public void start() throws MQClientException { this.defaultMQProducerImpl.initTransactionEnv(); diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByHash.java b/client/src/main/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByHash.java index 11e2822b225a3afb1ab535d623c0eb262cdcc1bb..ba8ea8b58a30ec86113ad962a0630b526c9b5b24 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByHash.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByHash.java @@ -25,12 +25,10 @@ public class SelectMessageQueueByHash implements MessageQueueSelector { @Override public MessageQueue select(List mqs, Message msg, Object arg) { - int value = arg.hashCode(); + int value = arg.hashCode() % mqs.size(); if (value < 0) { value = Math.abs(value); } - - value = value % mqs.size(); return mqs.get(value); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/stat/ConsumerStatsManager.java b/client/src/main/java/org/apache/rocketmq/client/stat/ConsumerStatsManager.java index cf347b4dd7e68c3b80d095662e7d447a3d0708ba..ba4773ae679355cced2d8bc4579aed0ec92b291d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/stat/ConsumerStatsManager.java +++ b/client/src/main/java/org/apache/rocketmq/client/stat/ConsumerStatsManager.java @@ -62,7 +62,7 @@ public class ConsumerStatsManager { } public void incPullRT(final String group, final String topic, final long rt) { - this.topicAndGroupPullRT.addValue(topic + "@" + group, (int) rt, 1); + this.topicAndGroupPullRT.addRTValue(topic + "@" + group, (int) rt, 1); } public void incPullTPS(final String group, final String topic, final long msgs) { @@ -70,7 +70,7 @@ public class ConsumerStatsManager { } public void incConsumeRT(final String group, final String topic, final long rt) { - this.topicAndGroupConsumeRT.addValue(topic + "@" + group, (int) rt, 1); + this.topicAndGroupConsumeRT.addRTValue(topic + "@" + group, (int) rt, 1); } public void incConsumeOKTPS(final String group, final String topic, final long msgs) { diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java index 8c3d886517595ce2fb53d1012735d8fb7af32b50..7ff8bd77e03152f2469db28ee90891ff1dd59be3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.client.trace; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -28,7 +27,9 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; + import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.AccessChannel; import org.apache.rocketmq.client.common.ThreadLocalIndex; @@ -41,11 +42,11 @@ import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.MessageQueueSelector; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; @@ -54,6 +55,7 @@ import static org.apache.rocketmq.client.trace.TraceConstants.TRACE_INSTANCE_NAM public class AsyncTraceDispatcher implements TraceDispatcher { private final static InternalLogger log = ClientLogger.getLog(); + private final static AtomicInteger COUNTER = new AtomicInteger(); private final int queueSize; private final int batchSize; private final int maxMsgSize; @@ -62,7 +64,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher { // The last discard number of log private AtomicLong discardCount; private Thread worker; - private ArrayBlockingQueue traceContextQueue; + private final ArrayBlockingQueue traceContextQueue; private ArrayBlockingQueue appenderQueue; private volatile Thread shutDownHook; private volatile boolean stopped = false; @@ -76,7 +78,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher { private String group; private Type type; - public AsyncTraceDispatcher(String group, Type type,String traceTopicName, RPCHook rpcHook) { + public AsyncTraceDispatcher(String group, Type type, String traceTopicName, RPCHook rpcHook) { // queueSize is greater than or equal to the n power of 2 of value this.queueSize = 2048; this.batchSize = 100; @@ -90,15 +92,15 @@ public class AsyncTraceDispatcher implements TraceDispatcher { if (!UtilAll.isBlank(traceTopicName)) { this.traceTopicName = traceTopicName; } else { - this.traceTopicName = MixAll.RMQ_SYS_TRACE_TOPIC; + this.traceTopicName = TopicValidator.RMQ_SYS_TRACE_TOPIC; } this.traceExecutor = new ThreadPoolExecutor(// - 10, // - 20, // - 1000 * 60, // - TimeUnit.MILLISECONDS, // - this.appenderQueue, // - new ThreadFactoryImpl("MQTraceSendThread_")); + 10, // + 20, // + 1000 * 60, // + TimeUnit.MILLISECONDS, // + this.appenderQueue, // + new ThreadFactoryImpl("MQTraceSendThread_")); traceProducer = getAndCreateTraceProducer(rpcHook); } @@ -165,7 +167,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher { } private String genGroupNameForTrace() { - return TraceConstants.GROUP_NAME_PREFIX + "-" + this.group + "-" + this.type ; + return TraceConstants.GROUP_NAME_PREFIX + "-" + this.group + "-" + this.type + "-" + COUNTER.incrementAndGet(); } @Override @@ -178,10 +180,15 @@ public class AsyncTraceDispatcher implements TraceDispatcher { } @Override - public void flush() throws IOException { + public void flush() { // The maximum waiting time for refresh,avoid being written all the time, resulting in failure to return. long end = System.currentTimeMillis() + 500; - while (traceContextQueue.size() > 0 || appenderQueue.size() > 0 && System.currentTimeMillis() <= end) { + while (System.currentTimeMillis() <= end) { + synchronized (traceContextQueue) { + if (traceContextQueue.size() == 0 && appenderQueue.size() == 0) { + break; + } + } try { Thread.sleep(1); } catch (InterruptedException e) { @@ -194,6 +201,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher { @Override public void shutdown() { this.stopped = true; + flush(); this.traceExecutor.shutdown(); if (isStarted.get()) { traceProducer.shutdown(); @@ -210,11 +218,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher { public void run() { synchronized (this) { if (!this.hasShutdown) { - try { - flush(); - } catch (IOException e) { - log.error("system MQTrace hook shutdown failed ,maybe loss some trace data"); - } + flush(); } } } @@ -240,25 +244,27 @@ public class AsyncTraceDispatcher implements TraceDispatcher { public void run() { while (!stopped) { List contexts = new ArrayList(batchSize); - for (int i = 0; i < batchSize; i++) { - TraceContext context = null; - try { - //get trace data element from blocking Queue — traceContextQueue - context = traceContextQueue.poll(5, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { + synchronized (traceContextQueue) { + for (int i = 0; i < batchSize; i++) { + TraceContext context = null; + try { + //get trace data element from blocking Queue - traceContextQueue + context = traceContextQueue.poll(5, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + } + if (context != null) { + contexts.add(context); + } else { + break; + } } - if (context != null) { - contexts.add(context); - } else { - break; + if (contexts.size() > 0) { + AsyncAppenderRequest request = new AsyncAppenderRequest(contexts); + traceExecutor.submit(request); + } else if (AsyncTraceDispatcher.this.stopped) { + this.stopped = true; } } - if (contexts.size() > 0) { - AsyncAppenderRequest request = new AsyncAppenderRequest(contexts); - traceExecutor.submit(request); - } else if (AsyncTraceDispatcher.this.stopped) { - this.stopped = true; - } } } @@ -350,7 +356,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher { * Send message trace data * * @param keySet the keyset in this batch(including msgId in original message not offsetMsgId) - * @param data the message trace data in this batch + * @param data the message trace data in this batch */ private void sendTraceDataByMQ(Set keySet, final String data, String dataTopic, String regionId) { String traceTopic = traceTopicName; @@ -370,7 +376,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher { @Override public void onException(Throwable e) { - log.info("send trace data ,the traceData is " + data); + log.error("send trace data failed, the traceData is {}", data, e); } }; if (traceBrokerSet.isEmpty()) { @@ -387,7 +393,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher { filterMqs.add(queue); } } - int index = sendWhichQueue.getAndIncrement(); + int index = sendWhichQueue.incrementAndGet(); int pos = Math.abs(index) % filterMqs.size(); if (pos < 0) { pos = 0; @@ -398,7 +404,7 @@ public class AsyncTraceDispatcher implements TraceDispatcher { } } catch (Exception e) { - log.info("send trace data,the traceData is" + data); + log.error("send trace data failed, the traceData is {}", data, e); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java index f93aa38b8293712b56a9af289c15d3586f67e76e..70c147e1eb349649cf76afc91b1bf661a619c7d1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.client.trace; +import org.apache.rocketmq.client.producer.LocalTransactionState; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageType; @@ -32,7 +33,9 @@ public class TraceBean { private int retryTimes; private int bodyLength; private MessageType msgType; - + private LocalTransactionState transactionState; + private String transactionId; + private boolean fromTransactionCheck; public MessageType getMsgType() { return msgType; @@ -141,4 +144,28 @@ public class TraceBean { public void setBodyLength(int bodyLength) { this.bodyLength = bodyLength; } + + public LocalTransactionState getTransactionState() { + return transactionState; + } + + public void setTransactionState(LocalTransactionState transactionState) { + this.transactionState = transactionState; + } + + public String getTransactionId() { + return transactionId; + } + + public void setTransactionId(String transactionId) { + this.transactionId = transactionId; + } + + public boolean isFromTransactionCheck() { + return fromTransactionCheck; + } + + public void setFromTransactionCheck(boolean fromTransactionCheck) { + this.fromTransactionCheck = fromTransactionCheck; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java index cb4a246517d62e711bd7b7dd171fb4e97d575482..1ad4b6105153d4dd4e167887dc54d3bee451bb56 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java @@ -16,7 +16,7 @@ */ package org.apache.rocketmq.client.trace; -import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.topic.TopicValidator; public class TraceConstants { @@ -24,5 +24,21 @@ public class TraceConstants { public static final char CONTENT_SPLITOR = (char) 1; public static final char FIELD_SPLITOR = (char) 2; public static final String TRACE_INSTANCE_NAME = "PID_CLIENT_INNER_TRACE_PRODUCER"; - public static final String TRACE_TOPIC_PREFIX = MixAll.SYSTEM_TOPIC_PREFIX + "TRACE_DATA_"; + public static final String TRACE_TOPIC_PREFIX = TopicValidator.SYSTEM_TOPIC_PREFIX + "TRACE_DATA_"; + public static final String TO_PREFIX = "To_"; + public static final String FROM_PREFIX = "From_"; + public static final String END_TRANSACTION = "EndTransaction"; + public static final String ROCKETMQ_SERVICE = "rocketmq"; + public static final String ROCKETMQ_SUCCESS = "rocketmq.success"; + public static final String ROCKETMQ_TAGS = "rocketmq.tags"; + public static final String ROCKETMQ_KEYS = "rocketmq.keys"; + public static final String ROCKETMQ_SOTRE_HOST = "rocketmq.store_host"; + public static final String ROCKETMQ_BODY_LENGTH = "rocketmq.body_length"; + public static final String ROCKETMQ_MSG_ID = "rocketmq.mgs_id"; + public static final String ROCKETMQ_MSG_TYPE = "rocketmq.mgs_type"; + public static final String ROCKETMQ_REGION_ID = "rocketmq.region_id"; + public static final String ROCKETMQ_TRANSACTION_ID = "rocketmq.transaction_id"; + public static final String ROCKETMQ_TRANSACTION_STATE = "rocketmq.transaction_state"; + public static final String ROCKETMQ_IS_FROM_TRANSACTION_CHECK = "rocketmq.is_from_transaction_check"; + public static final String ROCKETMQ_RETRY_TIMERS = "rocketmq.retry_times"; } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java index 5a1afaf3617a5922eb122a829e31f3ab3fbbe184..10aa2413591309fc72b17f090b7b84af548deb86 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.client.trace; +import org.apache.rocketmq.client.producer.LocalTransactionState; import org.apache.rocketmq.common.message.MessageType; import java.util.ArrayList; @@ -62,6 +63,14 @@ public class TraceDataEncoder { bean.setOffsetMsgId(line[12]); pubContext.setSuccess(Boolean.parseBoolean(line[13])); } + + // compatible with the old version + if (line.length >= 15) { + bean.setOffsetMsgId(line[12]); + pubContext.setSuccess(Boolean.parseBoolean(line[13])); + bean.setClientHost(line[14]); + } + pubContext.setTraceBeans(new ArrayList(1)); pubContext.getTraceBeans().add(bean); resList.add(pubContext); @@ -94,7 +103,32 @@ public class TraceDataEncoder { // add the context type subAfterContext.setContextCode(Integer.parseInt(line[6])); } + // compatible with the old version + if (line.length >= 9) { + subAfterContext.setTimeStamp(Long.parseLong(line[7])); + subAfterContext.setGroupName(line[8]); + } resList.add(subAfterContext); + } else if (line[0].equals(TraceType.EndTransaction.name())) { + TraceContext endTransactionContext = new TraceContext(); + endTransactionContext.setTraceType(TraceType.EndTransaction); + endTransactionContext.setTimeStamp(Long.parseLong(line[1])); + endTransactionContext.setRegionId(line[2]); + endTransactionContext.setGroupName(line[3]); + TraceBean bean = new TraceBean(); + bean.setTopic(line[4]); + bean.setMsgId(line[5]); + bean.setTags(line[6]); + bean.setKeys(line[7]); + bean.setStoreHost(line[8]); + bean.setMsgType(MessageType.values()[Integer.parseInt(line[9])]); + bean.setTransactionId(line[10]); + bean.setTransactionState(LocalTransactionState.valueOf(line[11])); + bean.setFromTransactionCheck(Boolean.parseBoolean(line[12])); + + endTransactionContext.setTraceBeans(new ArrayList(1)); + endTransactionContext.getTraceBeans().add(bean); + resList.add(endTransactionContext); } } return resList; @@ -130,7 +164,7 @@ public class TraceDataEncoder { .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getMsgType().ordinal()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getOffsetMsgId()).append(TraceConstants.CONTENT_SPLITOR)// - .append(ctx.isSuccess()).append(TraceConstants.FIELD_SPLITOR); + .append(ctx.isSuccess()).append(TraceConstants.FIELD_SPLITOR);// } break; case SubBefore: { @@ -155,9 +189,27 @@ public class TraceDataEncoder { .append(ctx.isSuccess()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)// .append(ctx.getContextCode()).append(TraceConstants.FIELD_SPLITOR); + } } break; + case EndTransaction: { + TraceBean bean = ctx.getTraceBeans().get(0); + sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getTopic()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getTags()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getStoreHost()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getMsgType().ordinal()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getTransactionId()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.getTransactionState().name()).append(TraceConstants.CONTENT_SPLITOR)// + .append(bean.isFromTransactionCheck()).append(TraceConstants.FIELD_SPLITOR); + } + break; default: } transferBean.setTransData(sb.toString()); diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java index 79b19c17e4e53e4213a456fb4862089d6a0a0920..8870ddcbdb3ef0e6a4ea3b35247891db30157e6d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java @@ -20,4 +20,5 @@ public enum TraceType { Pub, SubBefore, SubAfter, + EndTransaction, } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java new file mode 100644 index 0000000000000000000000000000000000000000..e78d37ab2cdbd4c5f0ece7d43aca814f1f44eb20 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.trace; + +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.codec.Charsets; +import org.apache.rocketmq.common.message.MessageExt; + +public class TraceView { + + private String msgId; + private String tags; + private String keys; + private String storeHost; + private String clientHost; + private int costTime; + private String msgType; + private String offSetMsgId; + private long timeStamp; + private long bornTime; + private String topic; + private String groupName; + private String status; + + public static List decodeFromTraceTransData(String key, MessageExt messageExt) { + List messageTraceViewList = new ArrayList(); + String messageBody = new String(messageExt.getBody(), Charsets.UTF_8); + if (messageBody == null || messageBody.length() <= 0) { + return messageTraceViewList; + } + + List traceContextList = TraceDataEncoder.decoderFromTraceDataString(messageBody); + + for (TraceContext context : traceContextList) { + TraceView messageTraceView = new TraceView(); + TraceBean traceBean = context.getTraceBeans().get(0); + if (!traceBean.getMsgId().equals(key)) { + continue; + } + messageTraceView.setCostTime(context.getCostTime()); + messageTraceView.setGroupName(context.getGroupName()); + if (context.isSuccess()) { + messageTraceView.setStatus("success"); + } else { + messageTraceView.setStatus("failed"); + } + messageTraceView.setKeys(traceBean.getKeys()); + messageTraceView.setMsgId(traceBean.getMsgId()); + messageTraceView.setTags(traceBean.getTags()); + messageTraceView.setTopic(traceBean.getTopic()); + messageTraceView.setMsgType(context.getTraceType().name()); + messageTraceView.setOffSetMsgId(traceBean.getOffsetMsgId()); + messageTraceView.setTimeStamp(context.getTimeStamp()); + messageTraceView.setStoreHost(traceBean.getStoreHost()); + messageTraceView.setClientHost(messageExt.getBornHostString()); + messageTraceViewList.add(messageTraceView); + } + return messageTraceViewList; + } + + public String getMsgId() { + return msgId; + } + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + public String getTags() { + return tags; + } + + public void setTags(String tags) { + this.tags = tags; + } + + public String getKeys() { + return keys; + } + + public void setKeys(String keys) { + this.keys = keys; + } + + public String getStoreHost() { + return storeHost; + } + + public void setStoreHost(String storeHost) { + this.storeHost = storeHost; + } + + public String getClientHost() { + return clientHost; + } + + public void setClientHost(String clientHost) { + this.clientHost = clientHost; + } + + public int getCostTime() { + return costTime; + } + + public void setCostTime(int costTime) { + this.costTime = costTime; + } + + public String getMsgType() { + return msgType; + } + + public void setMsgType(String msgType) { + this.msgType = msgType; + } + + public String getOffSetMsgId() { + return offSetMsgId; + } + + public void setOffSetMsgId(String offSetMsgId) { + this.offSetMsgId = offSetMsgId; + } + + public long getTimeStamp() { + return timeStamp; + } + + public void setTimeStamp(long timeStamp) { + this.timeStamp = timeStamp; + } + + public long getBornTime() { + return bornTime; + } + + public void setBornTime(long bornTime) { + this.bornTime = bornTime; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getGroupName() { + return groupName; + } + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageOpenTracingHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageOpenTracingHookImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..28fccae06f8d358925ff3e539ee0cf1c4c0ed52d --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageOpenTracingHookImpl.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace.hook; + +import io.opentracing.Span; +import io.opentracing.SpanContext; +import io.opentracing.Tracer; +import io.opentracing.propagation.Format; +import io.opentracing.propagation.TextMapAdapter; +import io.opentracing.tag.Tags; +import org.apache.rocketmq.client.hook.ConsumeMessageContext; +import org.apache.rocketmq.client.hook.ConsumeMessageHook; +import org.apache.rocketmq.client.trace.TraceConstants; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.NamespaceUtil; + +import java.util.ArrayList; +import java.util.List; + + +public class ConsumeMessageOpenTracingHookImpl implements ConsumeMessageHook { + + private Tracer tracer; + + public ConsumeMessageOpenTracingHookImpl(Tracer tracer) { + this.tracer = tracer; + } + + @Override + public String hookName() { + return "ConsumeMessageOpenTracingHook"; + } + + @Override + public void consumeMessageBefore(ConsumeMessageContext context) { + if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) { + return; + } + List spanList = new ArrayList<>(); + for (MessageExt msg : context.getMsgList()) { + if (msg == null) { + continue; + } + Tracer.SpanBuilder spanBuilder = tracer + .buildSpan(TraceConstants.FROM_PREFIX + msg.getTopic()) + .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_CONSUMER); + SpanContext spanContext = tracer.extract(Format.Builtin.TEXT_MAP, new TextMapAdapter(msg.getProperties())); + if (spanContext != null) { + spanBuilder.asChildOf(spanContext); + } + Span span = spanBuilder.start(); + + span.setTag(Tags.PEER_SERVICE, TraceConstants.ROCKETMQ_SERVICE); + span.setTag(Tags.MESSAGE_BUS_DESTINATION, NamespaceUtil.withoutNamespace(msg.getTopic())); + span.setTag(TraceConstants.ROCKETMQ_MSG_ID, msg.getMsgId()); + span.setTag(TraceConstants.ROCKETMQ_TAGS, msg.getTags()); + span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys()); + span.setTag(TraceConstants.ROCKETMQ_BODY_LENGTH, msg.getStoreSize()); + span.setTag(TraceConstants.ROCKETMQ_RETRY_TIMERS, msg.getReconsumeTimes()); + span.setTag(TraceConstants.ROCKETMQ_REGION_ID, msg.getProperty(MessageConst.PROPERTY_MSG_REGION)); + spanList.add(span); + } + context.setMqTraceContext(spanList); + } + + @Override + public void consumeMessageAfter(ConsumeMessageContext context) { + if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) { + return; + } + List spanList = (List) context.getMqTraceContext(); + if (spanList == null) { + return; + } + for (Span span : spanList) { + span.setTag(TraceConstants.ROCKETMQ_SUCCESS, context.isSuccess()); + span.finish(); + } + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java index f30b1211f9413c4531dcedcc228e491a4c8aeecc..bce613987ff65753ffca83a99ef2cbc9fadc11a6 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.client.trace.hook; +import java.util.Map; import org.apache.rocketmq.client.consumer.listener.ConsumeReturnType; import org.apache.rocketmq.client.hook.ConsumeMessageContext; import org.apache.rocketmq.client.hook.ConsumeMessageHook; @@ -91,7 +92,7 @@ public class ConsumeMessageTraceHookImpl implements ConsumeMessageHook { TraceContext subBeforeContext = (TraceContext) context.getMqTraceContext(); if (subBeforeContext.getTraceBeans() == null || subBeforeContext.getTraceBeans().size() < 1) { - // If subbefore bean is null ,skip it + // If subBefore bean is null ,skip it return; } TraceContext subAfterContext = new TraceContext(); @@ -101,13 +102,16 @@ public class ConsumeMessageTraceHookImpl implements ConsumeMessageHook { subAfterContext.setRequestId(subBeforeContext.getRequestId());// subAfterContext.setSuccess(context.isSuccess());// - // Caculate the cost time for processing messages + // Calculate the cost time for processing messages int costTime = (int) ((System.currentTimeMillis() - subBeforeContext.getTimeStamp()) / context.getMsgList().size()); subAfterContext.setCostTime(costTime);// subAfterContext.setTraceBeans(subBeforeContext.getTraceBeans()); - String contextType = context.getProps().get(MixAll.CONSUME_CONTEXT_TYPE); - if (contextType != null) { - subAfterContext.setContextCode(ConsumeReturnType.valueOf(contextType).ordinal()); + Map props = context.getProps(); + if (props != null) { + String contextType = props.get(MixAll.CONSUME_CONTEXT_TYPE); + if (contextType != null) { + subAfterContext.setContextCode(ConsumeReturnType.valueOf(contextType).ordinal()); + } } localDispatcher.append(subAfterContext); } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..62d310f1961f0971f11803367d9d83b04a8ee7b0 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace.hook; + +import io.opentracing.Span; +import io.opentracing.SpanContext; +import io.opentracing.Tracer; +import io.opentracing.propagation.Format; +import io.opentracing.propagation.TextMapAdapter; +import io.opentracing.tag.Tags; +import org.apache.rocketmq.client.hook.EndTransactionContext; +import org.apache.rocketmq.client.hook.EndTransactionHook; +import org.apache.rocketmq.client.trace.TraceConstants; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageType; + +public class EndTransactionOpenTracingHookImpl implements EndTransactionHook { + + private Tracer tracer; + + public EndTransactionOpenTracingHookImpl(Tracer tracer) { + this.tracer = tracer; + } + + @Override + public String hookName() { + return "EndTransactionOpenTracingHook"; + } + + @Override + public void endTransaction(EndTransactionContext context) { + if (context == null) { + return; + } + Message msg = context.getMessage(); + Tracer.SpanBuilder spanBuilder = tracer + .buildSpan(TraceConstants.END_TRANSACTION) + .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_PRODUCER); + SpanContext spanContext = tracer.extract(Format.Builtin.TEXT_MAP, new TextMapAdapter(msg.getProperties())); + if (spanContext != null) { + spanBuilder.asChildOf(spanContext); + } + + Span span = spanBuilder.start(); + span.setTag(Tags.PEER_SERVICE, TraceConstants.ROCKETMQ_SERVICE); + span.setTag(Tags.MESSAGE_BUS_DESTINATION, msg.getTopic()); + span.setTag(TraceConstants.ROCKETMQ_TAGS, msg.getTags()); + span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys()); + span.setTag(TraceConstants.ROCKETMQ_SOTRE_HOST, context.getBrokerAddr()); + span.setTag(TraceConstants.ROCKETMQ_MSG_ID, context.getMsgId()); + span.setTag(TraceConstants.ROCKETMQ_MSG_TYPE, MessageType.Trans_msg_Commit.name()); + span.setTag(TraceConstants.ROCKETMQ_TRANSACTION_ID, context.getTransactionId()); + span.setTag(TraceConstants.ROCKETMQ_TRANSACTION_STATE, context.getTransactionState().name()); + span.setTag(TraceConstants.ROCKETMQ_IS_FROM_TRANSACTION_CHECK, context.isFromTransactionCheck()); + span.finish(); + } + +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionTraceHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionTraceHookImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..cbd755ba39e19a65386ec65252a9f1f817e486fd --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionTraceHookImpl.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace.hook; + +import org.apache.rocketmq.client.hook.EndTransactionContext; +import org.apache.rocketmq.client.hook.EndTransactionHook; +import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; +import org.apache.rocketmq.client.trace.TraceBean; +import org.apache.rocketmq.client.trace.TraceContext; +import org.apache.rocketmq.client.trace.TraceDispatcher; +import org.apache.rocketmq.client.trace.TraceType; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageType; +import org.apache.rocketmq.common.protocol.NamespaceUtil; + +import java.util.ArrayList; + +public class EndTransactionTraceHookImpl implements EndTransactionHook { + + private TraceDispatcher localDispatcher; + + public EndTransactionTraceHookImpl(TraceDispatcher localDispatcher) { + this.localDispatcher = localDispatcher; + } + + @Override + public String hookName() { + return "EndTransactionTraceHook"; + } + + @Override + public void endTransaction(EndTransactionContext context) { + //if it is message trace data,then it doesn't recorded + if (context == null || context.getMessage().getTopic().startsWith(((AsyncTraceDispatcher) localDispatcher).getTraceTopicName())) { + return; + } + Message msg = context.getMessage(); + //build the context content of TuxeTraceContext + TraceContext tuxeContext = new TraceContext(); + tuxeContext.setTraceBeans(new ArrayList(1)); + tuxeContext.setTraceType(TraceType.EndTransaction); + tuxeContext.setGroupName(NamespaceUtil.withoutNamespace(context.getProducerGroup())); + //build the data bean object of message trace + TraceBean traceBean = new TraceBean(); + traceBean.setTopic(NamespaceUtil.withoutNamespace(context.getMessage().getTopic())); + traceBean.setTags(context.getMessage().getTags()); + traceBean.setKeys(context.getMessage().getKeys()); + traceBean.setStoreHost(context.getBrokerAddr()); + traceBean.setMsgType(MessageType.Trans_msg_Commit); + traceBean.setClientHost(((AsyncTraceDispatcher)localDispatcher).getHostProducer().getmQClientFactory().getClientId()); + traceBean.setMsgId(context.getMsgId()); + traceBean.setTransactionState(context.getTransactionState()); + traceBean.setTransactionId(context.getTransactionId()); + traceBean.setFromTransactionCheck(context.isFromTransactionCheck()); + String regionId = msg.getProperty(MessageConst.PROPERTY_MSG_REGION); + if (regionId == null || regionId.isEmpty()) { + regionId = MixAll.DEFAULT_TRACE_REGION_ID; + } + tuxeContext.setRegionId(regionId); + tuxeContext.getTraceBeans().add(traceBean); + tuxeContext.setTimeStamp(System.currentTimeMillis()); + localDispatcher.append(tuxeContext); + } + +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..60c18a22a77ad8f265ebfb38ca91c239dce8a01d --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace.hook; + +import io.opentracing.Span; +import io.opentracing.SpanContext; +import io.opentracing.Tracer; +import io.opentracing.propagation.Format; +import io.opentracing.propagation.TextMapAdapter; +import io.opentracing.tag.Tags; +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.hook.SendMessageHook; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.client.trace.TraceConstants; +import org.apache.rocketmq.common.message.Message; + +public class SendMessageOpenTracingHookImpl implements SendMessageHook { + + private Tracer tracer; + + public SendMessageOpenTracingHookImpl(Tracer tracer) { + this.tracer = tracer; + } + + @Override + public String hookName() { + return "SendMessageOpenTracingHook"; + } + + @Override + public void sendMessageBefore(SendMessageContext context) { + if (context == null) { + return; + } + Message msg = context.getMessage(); + Tracer.SpanBuilder spanBuilder = tracer + .buildSpan(TraceConstants.TO_PREFIX + msg.getTopic()) + .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_PRODUCER); + SpanContext spanContext = tracer.extract(Format.Builtin.TEXT_MAP, new TextMapAdapter(msg.getProperties())); + if (spanContext != null) { + spanBuilder.asChildOf(spanContext); + } + Span span = spanBuilder.start(); + tracer.inject(span.context(), Format.Builtin.TEXT_MAP, new TextMapAdapter(msg.getProperties())); + span.setTag(Tags.PEER_SERVICE, TraceConstants.ROCKETMQ_SERVICE); + span.setTag(Tags.MESSAGE_BUS_DESTINATION, msg.getTopic()); + span.setTag(TraceConstants.ROCKETMQ_TAGS, msg.getTags()); + span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys()); + span.setTag(TraceConstants.ROCKETMQ_SOTRE_HOST, context.getBrokerAddr()); + span.setTag(TraceConstants.ROCKETMQ_MSG_TYPE, context.getMsgType().name()); + span.setTag(TraceConstants.ROCKETMQ_BODY_LENGTH, msg.getBody().length); + context.setMqTraceContext(span); + } + + @Override + public void sendMessageAfter(SendMessageContext context) { + if (context == null || context.getMqTraceContext() == null) { + return; + } + if (context.getSendResult() == null) { + return; + } + + if (context.getSendResult().getRegionId() == null) { + return; + } + + Span span = (Span) context.getMqTraceContext(); + span.setTag(TraceConstants.ROCKETMQ_SUCCESS, context.getSendResult().getSendStatus().equals(SendStatus.SEND_OK)); + span.setTag(TraceConstants.ROCKETMQ_MSG_ID, context.getSendResult().getMsgId()); + span.setTag(TraceConstants.ROCKETMQ_REGION_ID, context.getSendResult().getRegionId()); + span.finish(); + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java b/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java index e2b9abd46f045bdefbc4f78bdc8112ac899b4563..343fe4bca69707da07858e84daea2d0927cd6722 100644 --- a/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/ValidatorsTest.java @@ -19,11 +19,12 @@ package org.apache.rocketmq.client; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.topic.TopicValidator; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; +import static org.junit.Assert.fail; public class ValidatorsTest { @@ -47,17 +48,6 @@ public class ValidatorsTest { } } - @Test - public void testCheckTopic_UseDefaultTopic() { - String defaultTopic = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC; - try { - Validators.checkTopic(defaultTopic); - failBecauseExceptionWasNotThrown(MQClientException.class); - } catch (MQClientException e) { - assertThat(e).hasMessageStartingWith(String.format("The topic[%s] is conflict with AUTO_CREATE_TOPIC_KEY_TOPIC.", defaultTopic)); - } - } - @Test public void testCheckTopic_BlankTopic() { String blankTopic = ""; @@ -80,4 +70,30 @@ public class ValidatorsTest { assertThat(e).hasMessageStartingWith("The specified topic is longer than topic max length"); } } + + @Test + public void testIsSystemTopic() { + for (String topic : TopicValidator.getSystemTopicSet()) { + try { + Validators.isSystemTopic(topic); + fail("excepted MQClientException for system topic"); + } catch (MQClientException e) { + assertThat(e.getResponseCode()).isEqualTo(-1); + assertThat(e.getErrorMessage()).isEqualTo(String.format("The topic[%s] is conflict with system topic.", topic)); + } + } + } + + @Test + public void testIsNotAllowedSendTopic() { + for (String topic : TopicValidator.getNotAllowedSendTopicSet()) { + try { + Validators.isNotAllowedSendTopic(topic); + fail("excepted MQClientException for blacklist topic"); + } catch (MQClientException e) { + assertThat(e.getResponseCode()).isEqualTo(-1); + assertThat(e.getErrorMessage()).isEqualTo(String.format("Sending message to topic[%s] is forbidden.", topic)); + } + } + } } diff --git a/client/src/test/java/org/apache/rocketmq/client/common/ThreadLocalIndexTest.java b/client/src/test/java/org/apache/rocketmq/client/common/ThreadLocalIndexTest.java index 1be93ce0f3a1dd6d557d841c85109f3410bc40c6..de35b9181b6024eb16b8678f9314da6bc95e4316 100644 --- a/client/src/test/java/org/apache/rocketmq/client/common/ThreadLocalIndexTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/common/ThreadLocalIndexTest.java @@ -22,11 +22,18 @@ import static org.assertj.core.api.Assertions.assertThat; public class ThreadLocalIndexTest { @Test - public void testGetAndIncrement() throws Exception { + public void testIncrementAndGet() throws Exception { ThreadLocalIndex localIndex = new ThreadLocalIndex(); - int initialVal = localIndex.getAndIncrement(); + int initialVal = localIndex.incrementAndGet(); - assertThat(localIndex.getAndIncrement()).isEqualTo(initialVal + 1); + assertThat(localIndex.incrementAndGet()).isEqualTo(initialVal + 1); + } + + @Test + public void testIncrementAndGet2() throws Exception { + ThreadLocalIndex localIndex = new ThreadLocalIndex(); + int initialVal = localIndex.incrementAndGet(); + assertThat(initialVal >= 0); } } \ No newline at end of file diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java index cc8d5e2bf78a8ba0c840c1ec4e1045b89f931009..04b760eec2281561a70c7723208f10d1125d457f 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java @@ -24,6 +24,8 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.consumer.store.ReadOffsetType; @@ -48,16 +50,15 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.remoting.RPCHook; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; @@ -71,8 +72,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -@RunWith(PowerMockRunner.class) -@PrepareForTest(DefaultLitePullConsumerImpl.class) +@RunWith(MockitoJUnitRunner.class) public class DefaultLitePullConsumerTest { @Spy private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); @@ -81,6 +81,8 @@ public class DefaultLitePullConsumerTest { private MQClientAPIImpl mQClientAPIImpl; @Mock private MQAdminImpl mQAdminImpl; + @Mock + private AssignedMessageQueue assignedMQ; private RebalanceImpl rebalanceImpl; private OffsetStore offsetStore; @@ -92,7 +94,10 @@ public class DefaultLitePullConsumerTest { @Before public void init() throws Exception { - PowerMockito.suppress(PowerMockito.method(DefaultLitePullConsumerImpl.class, "updateTopicSubscribeInfoWhenSubscriptionChanged")); + ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); + factoryTable.forEach((s, instance) -> instance.shutdown()); + factoryTable.clear(); + Field field = MQClientInstance.class.getDeclaredField("rebalanceService"); field.setAccessible(true); RebalanceService rebalanceService = (RebalanceService) field.get(mQClientFactory); @@ -180,7 +185,9 @@ public class DefaultLitePullConsumerTest { when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L); MessageQueue messageQueue = createMessageQueue(); - litePullConsumer.assign(Collections.singletonList(messageQueue)); + List messageQueues = Collections.singletonList(messageQueue); + litePullConsumer.assign(messageQueues); + litePullConsumer.pause(messageQueues); long offset = litePullConsumer.committed(messageQueue); litePullConsumer.seek(messageQueue, offset); Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); @@ -196,7 +203,9 @@ public class DefaultLitePullConsumerTest { when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L); MessageQueue messageQueue = createMessageQueue(); - litePullConsumer.assign(Collections.singletonList(messageQueue)); + List messageQueues = Collections.singletonList(messageQueue); + litePullConsumer.assign(messageQueues); + litePullConsumer.pause(messageQueues); litePullConsumer.seekToBegin(messageQueue); Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); field.setAccessible(true); @@ -211,7 +220,9 @@ public class DefaultLitePullConsumerTest { when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L); MessageQueue messageQueue = createMessageQueue(); - litePullConsumer.assign(Collections.singletonList(messageQueue)); + List messageQueues = Collections.singletonList(messageQueue); + litePullConsumer.assign(messageQueues); + litePullConsumer.pause(messageQueues); litePullConsumer.seekToEnd(messageQueue); Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); field.setAccessible(true); @@ -226,7 +237,9 @@ public class DefaultLitePullConsumerTest { when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(100L); MessageQueue messageQueue = createMessageQueue(); - litePullConsumer.assign(Collections.singletonList(messageQueue)); + List messageQueues = Collections.singletonList(messageQueue); + litePullConsumer.assign(messageQueues); + litePullConsumer.pause(messageQueues); try { litePullConsumer.seek(messageQueue, -1); failBecauseExceptionWasNotThrown(MQClientException.class); @@ -304,6 +317,53 @@ public class DefaultLitePullConsumerTest { } } + @Test + public void testPullTaskImpl_ProcessQueueNull() throws Exception { + DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + Field field = DefaultLitePullConsumer.class.getDeclaredField("defaultLitePullConsumerImpl"); + field.setAccessible(true); + // set ProcessQueue dropped = true + DefaultLitePullConsumerImpl localLitePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); + field.setAccessible(true); + when(assignedMQ.isPaused(any(MessageQueue.class))).thenReturn(false); + when(assignedMQ.getProcessQueue(any(MessageQueue.class))).thenReturn(null); + litePullConsumer.start(); + field.set(localLitePullConsumerImpl, assignedMQ); + + List result = litePullConsumer.poll(100); + assertThat(result.isEmpty()).isTrue(); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testPullTaskImpl_ProcessQueueDropped() throws Exception { + DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + Field field = DefaultLitePullConsumer.class.getDeclaredField("defaultLitePullConsumerImpl"); + field.setAccessible(true); + // set ProcessQueue dropped = true + DefaultLitePullConsumerImpl localLitePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); + field.setAccessible(true); + AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(localLitePullConsumerImpl); + assignedMessageQueue.getProcessQueue(messageQueue).setDropped(true); + litePullConsumer.start(); + + List result = litePullConsumer.poll(100); + assertThat(result.isEmpty()).isTrue(); + } finally { + litePullConsumer.shutdown(); + } + } + @Test public void testRegisterTopicMessageQueueChangeListener_Success() throws Exception { flag = false; @@ -464,6 +524,30 @@ public class DefaultLitePullConsumerTest { assertThat(offset).isEqualTo(100); } + @Test + public void testConsumerAfterShutdown() throws Exception { + DefaultLitePullConsumer defaultLitePullConsumer = createSubscribeLitePullConsumer(); + + new AsyncConsumer().executeAsync(defaultLitePullConsumer); + + Thread.sleep(100); + defaultLitePullConsumer.shutdown(); + assertThat(defaultLitePullConsumer.isRunning()).isFalse(); + } + + static class AsyncConsumer { + public void executeAsync(final DefaultLitePullConsumer consumer) { + new Thread(new Runnable() { + @Override + public void run() { + while (consumer.isRunning()) { + List poll = consumer.poll(2 * 1000); + } + } + }).start(); + } + } + private void initDefaultLitePullConsumer(DefaultLitePullConsumer litePullConsumer) throws Exception { Field field = DefaultLitePullConsumer.class.getDeclaredField("defaultLitePullConsumerImpl"); @@ -500,9 +584,9 @@ public class DefaultLitePullConsumerTest { when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) - .thenAnswer(new Answer() { + .thenAnswer(new Answer() { @Override - public Object answer(InvocationOnMock mock) throws Throwable { + public PullResult answer(InvocationOnMock mock) throws Throwable { PullMessageRequestHeader requestHeader = mock.getArgument(1); MessageClientExt messageClientExt = new MessageClientExt(); messageClientExt.setTopic(topic); @@ -528,6 +612,7 @@ public class DefaultLitePullConsumerTest { DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); litePullConsumer.subscribe(topic, "*"); + suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer); litePullConsumer.start(); initDefaultLitePullConsumer(litePullConsumer); return litePullConsumer; @@ -536,6 +621,7 @@ public class DefaultLitePullConsumerTest { private DefaultLitePullConsumer createStartLitePullConsumer() throws Exception { DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer); litePullConsumer.start(); initDefaultLitePullConsumer(litePullConsumer); return litePullConsumer; @@ -551,6 +637,7 @@ public class DefaultLitePullConsumerTest { litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); litePullConsumer.setMessageModel(MessageModel.BROADCASTING); litePullConsumer.subscribe(topic, "*"); + suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer); litePullConsumer.start(); initDefaultLitePullConsumer(litePullConsumer); return litePullConsumer; @@ -572,4 +659,15 @@ public class DefaultLitePullConsumerTest { } return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); } + + private static void suppressUpdateTopicRouteInfoFromNameServer(DefaultLitePullConsumer litePullConsumer) throws IllegalAccessException { + DefaultLitePullConsumerImpl defaultLitePullConsumerImpl = (DefaultLitePullConsumerImpl) FieldUtils.readDeclaredField(litePullConsumer, "defaultLitePullConsumerImpl", true); + if (litePullConsumer.getMessageModel() == MessageModel.CLUSTERING) { + litePullConsumer.changeInstanceNameToPID(); + } + MQClientInstance mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(litePullConsumer, (RPCHook) FieldUtils.readDeclaredField(defaultLitePullConsumerImpl, "rpcHook", true))); + ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); + factoryTable.put(litePullConsumer.buildMQClientId(), mQClientFactory); + doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); + } } diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index e6f0e866882ec460ba4505d007fca8e0ebfd995c..6c8463542fa61e00e7f84fa980c5588dd827ceee 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -17,14 +17,17 @@ package org.apache.rocketmq.client.consumer; import java.io.ByteArrayOutputStream; -import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; @@ -36,14 +39,15 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService; import org.apache.rocketmq.client.impl.consumer.ConsumeMessageOrderlyService; import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; import org.apache.rocketmq.client.impl.consumer.ProcessQueue; -import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; import org.apache.rocketmq.client.impl.consumer.PullMessageService; import org.apache.rocketmq.client.impl.consumer.PullRequest; import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.impl.consumer.RebalanceImpl; import org.apache.rocketmq.client.impl.consumer.RebalancePushImpl; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.common.message.MessageClientExt; @@ -51,6 +55,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.junit.After; import org.junit.Assert; @@ -59,10 +64,8 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; @@ -72,11 +75,11 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -@RunWith(PowerMockRunner.class) -@PrepareForTest(DefaultMQPushConsumerImpl.class) +@RunWith(MockitoJUnitRunner.class) public class DefaultMQPushConsumerTest { private String consumerGroup; private String topic = "FooBar"; @@ -85,57 +88,21 @@ public class DefaultMQPushConsumerTest { @Mock private MQClientAPIImpl mQClientAPIImpl; - private PullAPIWrapper pullAPIWrapper; + private RebalanceImpl rebalanceImpl; private RebalancePushImpl rebalancePushImpl; private DefaultMQPushConsumer pushConsumer; @Before public void init() throws Exception { - consumerGroup = "FooBarGroup" + System.currentTimeMillis(); - pushConsumer = new DefaultMQPushConsumer(consumerGroup); - pushConsumer.setNamesrvAddr("127.0.0.1:9876"); - pushConsumer.setPullInterval(60 * 1000); - - pushConsumer.registerMessageListener(new MessageListenerConcurrently() { - @Override - public ConsumeConcurrentlyStatus consumeMessage(List msgs, - ConsumeConcurrentlyContext context) { - return null; - } - }); - - DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl(); - PowerMockito.suppress(PowerMockito.method(DefaultMQPushConsumerImpl.class, "updateTopicSubscribeInfoWhenSubscriptionChanged")); - rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl())); - Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); - field.setAccessible(true); - field.set(pushConsumerImpl, rebalancePushImpl); - - pushConsumer.subscribe(topic, "*"); - pushConsumer.start(); - - mQClientFactory = spy(pushConsumerImpl.getmQClientFactory()); - field = DefaultMQPushConsumerImpl.class.getDeclaredField("mQClientFactory"); - field.setAccessible(true); - field.set(pushConsumerImpl, mQClientFactory); + ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); + factoryTable.forEach((s, instance) -> instance.shutdown()); + factoryTable.clear(); - field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); - field.setAccessible(true); - field.set(mQClientFactory, mQClientAPIImpl); - - pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false)); - field = DefaultMQPushConsumerImpl.class.getDeclaredField("pullAPIWrapper"); - field.setAccessible(true); - field.set(pushConsumerImpl, pullAPIWrapper); - - pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().setmQClientFactory(mQClientFactory); - mQClientFactory.registerConsumer(consumerGroup, pushConsumerImpl); - - when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), + when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) - .thenAnswer(new Answer() { + .thenAnswer(new Answer() { @Override - public Object answer(InvocationOnMock mock) throws Throwable { + public PullResult answer(InvocationOnMock mock) throws Throwable { PullMessageRequestHeader requestHeader = mock.getArgument(1); MessageClientExt messageClientExt = new MessageClientExt(); messageClientExt.setTopic(topic); @@ -151,12 +118,43 @@ public class DefaultMQPushConsumerTest { } }); + + consumerGroup = "FooBarGroup" + System.currentTimeMillis(); + pushConsumer = new DefaultMQPushConsumer(consumerGroup); + pushConsumer.setNamesrvAddr("127.0.0.1:9876"); + pushConsumer.setPullInterval(60 * 1000); + + pushConsumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + return null; + } + }); + + DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl(); + rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl())); + + // suppress updateTopicRouteInfoFromNameServer + pushConsumer.changeInstanceNameToPID(); + mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true)); + FieldUtils.writeDeclaredField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true); + mQClientFactory = spy(mQClientFactory); + factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory); + doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); + doReturn(new FindBrokerResult("127.0.0.1:10911", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean()); - doReturn(Collections.singletonList(mQClientFactory.getClientId())).when(mQClientFactory).findConsumerIdList(anyString(), anyString()); + + rebalanceImpl = spy(pushConsumerImpl.getRebalanceImpl()); + doReturn(123L).when(rebalanceImpl).computePullFromWhereWithException(any(MessageQueue.class)); + FieldUtils.writeDeclaredField(pushConsumerImpl, "rebalanceImpl", rebalanceImpl, true); + Set messageQueueSet = new HashSet(); messageQueueSet.add(createPullRequest().getMessageQueue()); - pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet); - doReturn(123L).when(rebalancePushImpl).computePullFromWhere(any(MessageQueue.class)); + pushConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + + pushConsumer.subscribe(topic, "*"); + pushConsumer.start(); } @After @@ -172,12 +170,12 @@ public class DefaultMQPushConsumerTest { @Test public void testPullMessage_Success() throws InterruptedException, RemotingException, MQBrokerException { final CountDownLatch countDownLatch = new CountDownLatch(1); - final MessageExt[] messageExts = new MessageExt[1]; + final AtomicReference messageAtomic = new AtomicReference<>(); pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { - messageExts[0] = msgs.get(0); + messageAtomic.set(msgs.get(0)); countDownLatch.countDown(); return null; } @@ -185,20 +183,22 @@ public class DefaultMQPushConsumerTest { PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); pullMessageService.executePullRequestImmediately(createPullRequest()); - countDownLatch.await(); - assertThat(messageExts[0].getTopic()).isEqualTo(topic); - assertThat(messageExts[0].getBody()).isEqualTo(new byte[] {'a'}); + countDownLatch.await(10, TimeUnit.SECONDS); + MessageExt msg = messageAtomic.get(); + assertThat(msg).isNotNull(); + assertThat(msg.getTopic()).isEqualTo(topic); + assertThat(msg.getBody()).isEqualTo(new byte[] {'a'}); } @Test public void testPullMessage_SuccessWithOrderlyService() throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); - final MessageExt[] messageExts = new MessageExt[1]; + final AtomicReference messageAtomic = new AtomicReference<>(); MessageListenerOrderly listenerOrderly = new MessageListenerOrderly() { @Override public ConsumeOrderlyStatus consumeMessage(List msgs, ConsumeOrderlyContext context) { - messageExts[0] = msgs.get(0); + messageAtomic.set(msgs.get(0)); countDownLatch.countDown(); return null; } @@ -211,8 +211,10 @@ public class DefaultMQPushConsumerTest { pullMessageService.executePullRequestLater(createPullRequest(), 100); countDownLatch.await(10, TimeUnit.SECONDS); - assertThat(messageExts[0].getTopic()).isEqualTo(topic); - assertThat(messageExts[0].getBody()).isEqualTo(new byte[] {'a'}); + MessageExt msg = messageAtomic.get(); + assertThat(msg).isNotNull(); + assertThat(msg.getTopic()).isEqualTo(topic); + assertThat(msg.getBody()).isEqualTo(new byte[] {'a'}); } @Test @@ -256,6 +258,34 @@ public class DefaultMQPushConsumerTest { } } + @Test + public void testGracefulShutdown() throws InterruptedException, RemotingException, MQBrokerException, MQClientException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + pushConsumer.setAwaitTerminationMillisWhenShutdown(2000); + final AtomicBoolean messageConsumedFlag = new AtomicBoolean(false); + pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + countDownLatch.countDown(); + try { + Thread.sleep(1000); + messageConsumedFlag.set(true); + } catch (InterruptedException e) { + } + + return null; + } + })); + + PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); + pullMessageService.executePullRequestImmediately(createPullRequest()); + assertThat(countDownLatch.await(10, TimeUnit.SECONDS)).isTrue(); + + pushConsumer.shutdown(); + assertThat(messageConsumedFlag.get()).isTrue(); + } + private DefaultMQPushConsumer createPushConsumer() { DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(consumerGroup); pushConsumer.registerMessageListener(new MessageListenerConcurrently() { @@ -294,4 +324,21 @@ public class DefaultMQPushConsumerTest { } return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); } + + @Test + public void testPullMessage_ExceptionOccursWhenComputePullFromWhere() throws MQClientException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + final MessageExt[] messageExts = new MessageExt[1]; + pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService( + new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), + (msgs, context) -> { + messageExts[0] = msgs.get(0); + return null; + })); + + pushConsumer.getDefaultMQPushConsumerImpl().setConsumeOrderly(true); + PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); + pullMessageService.executePullRequestImmediately(createPullRequest()); + assertThat(messageExts[0]).isNull(); + } } diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStoreTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStoreTest.java index 64d64f29a9489fbc77577c71170e552718be3263..f762910a40118cd7b99c9c5c943d5a9ba226954b 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStoreTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStoreTest.java @@ -84,7 +84,7 @@ public class RemoteBrokerOffsetStoreTest { offsetStore.updateOffset(messageQueue, 1024, false); - doThrow(new MQBrokerException(-1, "")) + doThrow(new MQBrokerException(-1, "", null)) .when(mqClientAPI).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong()); assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-1); diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java index 921743c6be864401c6ba4aefb7e087cb26d4c492..e8feb80dd99f45f4713605f797130fba6e75f9e5 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java @@ -16,7 +16,19 @@ */ package org.apache.rocketmq.client.impl.consumer; -import org.apache.rocketmq.client.consumer.*; +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; @@ -40,23 +52,15 @@ import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; -import java.io.ByteArrayOutputStream; -import java.lang.reflect.Field; -import java.net.InetSocketAddress; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CountDownLatch; - import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.*; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; @@ -117,9 +121,9 @@ public class ConsumeMessageConcurrentlyServiceTest { when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) - .thenAnswer(new Answer() { + .thenAnswer(new Answer() { @Override - public Object answer(InvocationOnMock mock) throws Throwable { + public PullResult answer(InvocationOnMock mock) throws Throwable { PullMessageRequestHeader requestHeader = mock.getArgument(1); MessageClientExt messageClientExt = new MessageClientExt(); messageClientExt.setTopic(topic); @@ -145,13 +149,13 @@ public class ConsumeMessageConcurrentlyServiceTest { @Test public void testPullMessage_ConsumeSuccess() throws InterruptedException, RemotingException, MQBrokerException, NoSuchFieldException,Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); - final MessageExt[] messageExts = new MessageExt[1]; + final AtomicReference messageAtomic = new AtomicReference<>(); ConsumeMessageConcurrentlyService normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { - messageExts[0] = msgs.get(0); + messageAtomic.set(msgs.get(0)); countDownLatch.countDown(); return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } @@ -175,8 +179,10 @@ public class ConsumeMessageConcurrentlyServiceTest { StatsItem item = itemSet.getAndCreateStatsItem(topic + "@" + pushConsumer.getDefaultMQPushConsumerImpl().groupName()); assertThat(item.getValue().get()).isGreaterThan(0L); - assertThat(messageExts[0].getTopic()).isEqualTo(topic); - assertThat(messageExts[0].getBody()).isEqualTo(new byte[] {'a'}); + MessageExt msg = messageAtomic.get(); + assertThat(msg).isNotNull(); + assertThat(msg.getTopic()).isEqualTo(topic); + assertThat(msg.getBody()).isEqualTo(new byte[] {'a'}); } @After diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java index bb2132111c179e8fb13dc283aa4634f2313e6040..d37844ca9631ed801779571fd193bcee6135a20b 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java @@ -19,29 +19,44 @@ package org.apache.rocketmq.client.impl.factory; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.admin.MQAdminExtInner; import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.consumer.MQConsumerInner; import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.remoting.exception.RemotingException; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class MQClientInstanceTest { private MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); private String topic = "FooBar"; private String group = "FooBarGroup"; + private ConcurrentMap> brokerAddrTable = new ConcurrentHashMap>(); + + @Before + public void init() throws Exception { + FieldUtils.writeDeclaredField(mqClientInstance, "brokerAddrTable", brokerAddrTable, true); + } @Test public void testTopicRouteData2TopicPublishInfo() { @@ -64,7 +79,7 @@ public class MQClientInstanceTest { queueData.setPerm(6); queueData.setReadQueueNums(3); queueData.setWriteQueueNums(4); - queueData.setTopicSynFlag(0); + queueData.setTopicSysFlag(0); queueDataList.add(queueData); topicRouteData.setQueueDatas(queueDataList); @@ -74,6 +89,34 @@ public class MQClientInstanceTest { assertThat(topicPublishInfo.getMessageQueueList().size()).isEqualTo(4); } + @Test + public void testFindBrokerAddressInSubscribe() { + // dledger normal case + String brokerName = "BrokerA"; + HashMap addrMap = new HashMap(); + addrMap.put(0L, "127.0.0.1:10911"); + addrMap.put(1L, "127.0.0.1:10912"); + addrMap.put(2L, "127.0.0.1:10913"); + brokerAddrTable.put(brokerName, addrMap); + long brokerId = 1; + FindBrokerResult brokerResult = mqClientInstance.findBrokerAddressInSubscribe(brokerName, brokerId, false); + assertThat(brokerResult).isNotNull(); + assertThat(brokerResult.getBrokerAddr()).isEqualTo("127.0.0.1:10912"); + assertThat(brokerResult.isSlave()).isTrue(); + + // dledger case, when node n0 was voted as the leader + brokerName = "BrokerB"; + HashMap addrMapNew = new HashMap(); + addrMapNew.put(0L, "127.0.0.1:10911"); + addrMapNew.put(2L, "127.0.0.1:10912"); + addrMapNew.put(3L, "127.0.0.1:10913"); + brokerAddrTable.put(brokerName, addrMapNew); + brokerResult = mqClientInstance.findBrokerAddressInSubscribe(brokerName, brokerId, false); + assertThat(brokerResult).isNotNull(); + assertThat(brokerResult.getBrokerAddr()).isEqualTo("127.0.0.1:10912"); + assertThat(brokerResult.isSlave()).isTrue(); + } + @Test public void testRegisterProducer() { boolean flag = mqClientInstance.registerProducer(group, mock(DefaultMQProducerImpl.class)); @@ -100,6 +143,31 @@ public class MQClientInstanceTest { assertThat(flag).isTrue(); } + + @Test + public void testConsumerRunningInfoWhenConsumersIsEmptyOrNot() throws RemotingException, InterruptedException, MQBrokerException { + MQConsumerInner mockConsumerInner = mock(MQConsumerInner.class); + ConsumerRunningInfo mockConsumerRunningInfo = mock(ConsumerRunningInfo.class); + when(mockConsumerInner.consumerRunningInfo()).thenReturn(mockConsumerRunningInfo); + when(mockConsumerInner.consumeType()).thenReturn(ConsumeType.CONSUME_PASSIVELY); + Properties properties = new Properties(); + when(mockConsumerRunningInfo.getProperties()).thenReturn(properties); + mqClientInstance.unregisterConsumer(group); + + ConsumerRunningInfo runningInfo = mqClientInstance.consumerRunningInfo(group); + assertThat(runningInfo).isNull(); + boolean flag = mqClientInstance.registerConsumer(group, mockConsumerInner); + assertThat(flag).isTrue(); + + runningInfo = mqClientInstance.consumerRunningInfo(group); + assertThat(runningInfo).isNotNull(); + assertThat(mockConsumerInner.consumerRunningInfo().getProperties().get(ConsumerRunningInfo.PROP_CONSUME_TYPE)); + + mqClientInstance.unregisterConsumer(group); + flag = mqClientInstance.registerConsumer(group, mock(MQConsumerInner.class)); + assertThat(flag).isTrue(); + } + @Test public void testRegisterAdminExt() { boolean flag = mqClientInstance.registerAdminExt(group, mock(MQAdminExtInner.class)); diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 41046fcdf52cca9b675825fa51f370f4292df9d9..5f29fe113c5690cde47833638612ab2201f0ada4 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -232,6 +232,51 @@ public class DefaultMQProducerTest { countDownLatch.await(3000L, TimeUnit.MILLISECONDS); assertThat(cc.get()).isEqualTo(5); } + + @Test + public void testBatchSendMessageAsync() + throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + final AtomicInteger cc = new AtomicInteger(0); + final CountDownLatch countDownLatch = new CountDownLatch(4); + + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + SendCallback sendCallback = new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + countDownLatch.countDown(); + } + + @Override + public void onException(Throwable e) { + e.printStackTrace(); + cc.incrementAndGet(); + countDownLatch.countDown(); + } + }; + MessageQueueSelector messageQueueSelector = new MessageQueueSelector() { + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + return null; + } + }; + + List msgs = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Message message = new Message(); + message.setTopic("test"); + message.setBody(("hello world" + i).getBytes()); + msgs.add(message); + } + producer.send(msgs, sendCallback); + producer.send(msgs, sendCallback, 1000); + MessageQueue mq = new MessageQueue("test", "BrokerA", 1); + producer.send(msgs, mq, sendCallback); + // this message is send failed + producer.send(msgs, new MessageQueue(), sendCallback, 1000); + + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + assertThat(cc.get()).isEqualTo(1); + } @Test public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { @@ -436,7 +481,7 @@ public class DefaultMQProducerTest { queueData.setPerm(6); queueData.setReadQueueNums(3); queueData.setWriteQueueNums(4); - queueData.setTopicSynFlag(0); + queueData.setTopicSysFlag(0); queueDataList.add(queueData); topicRouteData.setQueueDatas(queueDataList); return topicRouteData; diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByHashTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByHashTest.java index 056e910bb806732de9aa1d51804c7097fb7d0511..8f286ee42952230e9136ed2e123475629539db6e 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByHashTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByHashTest.java @@ -44,6 +44,14 @@ public class SelectMessageQueueByHashTest { String anotherOrderId = "234"; MessageQueue selected = selector.select(messageQueues, message, orderId); assertThat(selector.select(messageQueues, message, anotherOrderId)).isNotEqualTo(selected); + + //No exception is thrown while order Id hashcode is Integer.MIN + anotherOrderId = "polygenelubricants"; + selector.select(messageQueues, message, anotherOrderId); + anotherOrderId = "GydZG_"; + selector.select(messageQueues, message, anotherOrderId); + anotherOrderId = "DESIGNING WORKHOUSES"; + selector.select(messageQueues, message, anotherOrderId); } } \ No newline at end of file diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueRetryTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueRetryTest.java new file mode 100644 index 0000000000000000000000000000000000000000..36a57c9b0655bc7eafd981b407b8b8f813430acc --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueRetryTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.producer.selector; + +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SelectMessageQueueRetryTest { + + private String topic = "TEST"; + + @Test + public void testSelect() throws Exception { + + TopicPublishInfo topicPublishInfo = new TopicPublishInfo(); + List messageQueueList = new ArrayList(); + for (int i = 0; i < 3; i++) { + MessageQueue mq = new MessageQueue(); + mq.setBrokerName("broker-" + i); + mq.setQueueId(0); + mq.setTopic(topic); + messageQueueList.add(mq); + } + + topicPublishInfo.setMessageQueueList(messageQueueList); + + Set retryBrokerNameSet = retryBroker(topicPublishInfo); + //always in Set （broker-0，broker-1，broker-2） + assertThat(retryBroker(topicPublishInfo)).isEqualTo(retryBrokerNameSet); + } + + private Set retryBroker(TopicPublishInfo topicPublishInfo) { + MessageQueue mqTmp = null; + Set retryBrokerNameSet = new HashSet(); + for (int times = 0; times < 3; times++) { + String lastBrokerName = null == mqTmp ? null : mqTmp.getBrokerName(); + mqTmp = topicPublishInfo.selectOneMessageQueue(lastBrokerName); + retryBrokerNameSet.add(mqTmp.getBrokerName()); + } + return retryBrokerNameSet; + } + +} \ No newline at end of file diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java new file mode 100644 index 0000000000000000000000000000000000000000..ecf72ae44cfefdb682346d366b557b0a322f4077 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.trace; + +import io.opentracing.mock.MockSpan; +import io.opentracing.mock.MockTracer; +import io.opentracing.tag.Tags; +import java.io.ByteArrayOutputStream; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.consumer.store.ReadOffsetType; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService; +import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; +import org.apache.rocketmq.client.impl.consumer.ProcessQueue; +import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; +import org.apache.rocketmq.client.impl.consumer.PullMessageService; +import org.apache.rocketmq.client.impl.consumer.PullRequest; +import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.impl.consumer.RebalancePushImpl; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.trace.hook.ConsumeMessageOpenTracingHookImpl; +import org.apache.rocketmq.common.message.MessageClientExt; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultMQConsumerWithOpenTracingTest { + private String consumerGroup; + + private String topic = "FooBar"; + private String brokerName = "BrokerA"; + private MQClientInstance mQClientFactory; + + @Mock + private MQClientAPIImpl mQClientAPIImpl; + private PullAPIWrapper pullAPIWrapper; + private RebalancePushImpl rebalancePushImpl; + private DefaultMQPushConsumer pushConsumer; + private MockTracer tracer = new MockTracer(); + + @Before + public void init() throws Exception { + ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); + factoryTable.forEach((s, instance) -> instance.shutdown()); + factoryTable.clear(); + + when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class), + anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) + .thenAnswer(new Answer() { + @Override + public PullResult answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + MessageClientExt messageClientExt = new MessageClientExt(); + messageClientExt.setTopic(topic); + messageClientExt.setQueueId(0); + messageClientExt.setMsgId("123"); + messageClientExt.setBody(new byte[]{'a'}); + messageClientExt.setOffsetMsgId("234"); + messageClientExt.setBornHost(new InetSocketAddress(8080)); + messageClientExt.setStoreHost(new InetSocketAddress(8080)); + PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); + ((PullCallback) mock.getArgument(4)).onSuccess(pullResult); + return pullResult; + } + }); + + + consumerGroup = "FooBarGroup" + System.currentTimeMillis(); + pushConsumer = new DefaultMQPushConsumer(consumerGroup); + pushConsumer.getDefaultMQPushConsumerImpl().registerConsumeMessageHook( + new ConsumeMessageOpenTracingHookImpl(tracer)); + pushConsumer.setNamesrvAddr("127.0.0.1:9876"); + pushConsumer.setPullInterval(60 * 1000); + + OffsetStore offsetStore = Mockito.mock(OffsetStore.class); + Mockito.when(offsetStore.readOffset(any(MessageQueue.class), any(ReadOffsetType.class))).thenReturn(0L); + pushConsumer.setOffsetStore(offsetStore); + + pushConsumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + return null; + } + }); + + DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl(); + + // suppress updateTopicRouteInfoFromNameServer + pushConsumer.changeInstanceNameToPID(); + mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true)); + FieldUtils.writeDeclaredField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true); + mQClientFactory = spy(mQClientFactory); + factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory); + doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); + + doReturn(new FindBrokerResult("127.0.0.1:10911", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean()); + + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createPullRequest().getMessageQueue()); + pushConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + + pushConsumer.subscribe(topic, "*"); + pushConsumer.start(); + } + + @After + public void terminate() { + pushConsumer.shutdown(); + } + + @Test + public void testPullMessage_WithTrace_Success() throws InterruptedException, RemotingException, MQBrokerException, MQClientException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + final AtomicReference messageAtomic = new AtomicReference<>(); + pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + messageAtomic.set(msgs.get(0)); + countDownLatch.countDown(); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + })); + + PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); + pullMessageService.executePullRequestImmediately(createPullRequest()); + countDownLatch.await(30, TimeUnit.SECONDS); + MessageExt msg = messageAtomic.get(); + assertThat(msg).isNotNull(); + assertThat(msg.getTopic()).isEqualTo(topic); + assertThat(msg.getBody()).isEqualTo(new byte[]{'a'}); + + // wait until consumeMessageAfter hook of tracer is done surely. + waitAtMost(1, TimeUnit.SECONDS).until(() -> tracer.finishedSpans().size() == 1); + MockSpan span = tracer.finishedSpans().get(0); + assertThat(span.tags().get(Tags.MESSAGE_BUS_DESTINATION.getKey())).isEqualTo(topic); + assertThat(span.tags().get(Tags.SPAN_KIND.getKey())).isEqualTo(Tags.SPAN_KIND_CONSUMER); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_SUCCESS)).isEqualTo(true); + } + + private PullRequest createPullRequest() { + PullRequest pullRequest = new PullRequest(); + pullRequest.setConsumerGroup(consumerGroup); + pullRequest.setNextOffset(1024); + + MessageQueue messageQueue = new MessageQueue(); + messageQueue.setBrokerName(brokerName); + messageQueue.setQueueId(0); + messageQueue.setTopic(topic); + pullRequest.setMessageQueue(messageQueue); + ProcessQueue processQueue = new ProcessQueue(); + processQueue.setLocked(true); + processQueue.setLastLockTimestamp(System.currentTimeMillis()); + pullRequest.setProcessQueue(processQueue); + + return pullRequest; + } + + private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, + List messageExtList) throws Exception { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (MessageExt messageExt : messageExtList) { + outputStream.write(MessageDecoder.encode(messageExt, false)); + } + return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); + } + +} diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java index 496c5143e02387f5a39289f8a77ce61021fbd927..aec7d2cb0e28e57884bb468b41d831ac23ece99b 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java @@ -26,8 +26,11 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.consumer.PullCallback; import org.apache.rocketmq.client.consumer.PullResult; @@ -40,6 +43,7 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService; import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; import org.apache.rocketmq.client.impl.consumer.ProcessQueue; @@ -53,16 +57,16 @@ import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageClientExt; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; - import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.junit.After; import org.junit.Before; @@ -70,10 +74,8 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -85,12 +87,11 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -@RunWith(PowerMockRunner.class) -@PrepareForTest(DefaultMQPushConsumerImpl.class) +@RunWith(MockitoJUnitRunner.class) public class DefaultMQConsumerWithTraceTest { private String consumerGroup; private String consumerGroupNormal; - private String producerGroupTraceTemp = MixAll.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); + private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); private String topic = "FooBar"; private String brokerName = "BrokerA"; @@ -113,6 +114,10 @@ public class DefaultMQConsumerWithTraceTest { @Before public void init() throws Exception { + ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); + factoryTable.forEach((s, instance) -> instance.shutdown()); + factoryTable.clear(); + consumerGroup = "FooBarGroup" + System.currentTimeMillis(); pushConsumer = new DefaultMQPushConsumer(consumerGroup, true, ""); consumerGroupNormal = "FooBarGroup" + System.currentTimeMillis(); @@ -132,8 +137,14 @@ public class DefaultMQConsumerWithTraceTest { } }); - PowerMockito.suppress(PowerMockito.method(DefaultMQPushConsumerImpl.class, "updateTopicSubscribeInfoWhenSubscriptionChanged")); DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl(); + + // suppress updateTopicRouteInfoFromNameServer + pushConsumer.changeInstanceNameToPID(); + mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true))); + factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory); + doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); + rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl())); Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); field.setAccessible(true); @@ -171,9 +182,9 @@ public class DefaultMQConsumerWithTraceTest { when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) - .thenAnswer(new Answer() { + .thenAnswer(new Answer() { @Override - public Object answer(InvocationOnMock mock) throws Throwable { + public PullResult answer(InvocationOnMock mock) throws Throwable { PullMessageRequestHeader requestHeader = mock.getArgument(1); MessageClientExt messageClientExt = new MessageClientExt(); messageClientExt.setTopic(topic); @@ -205,12 +216,12 @@ public class DefaultMQConsumerWithTraceTest { traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); final CountDownLatch countDownLatch = new CountDownLatch(1); - final MessageExt[] messageExts = new MessageExt[1]; + final AtomicReference messageAtomic = new AtomicReference<>(); pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { - messageExts[0] = msgs.get(0); + messageAtomic.set(msgs.get(0)); countDownLatch.countDown(); return null; } @@ -218,9 +229,11 @@ public class DefaultMQConsumerWithTraceTest { PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); pullMessageService.executePullRequestImmediately(createPullRequest()); - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); - assertThat(messageExts[0].getTopic()).isEqualTo(topic); - assertThat(messageExts[0].getBody()).isEqualTo(new byte[] {'a'}); + countDownLatch.await(30, TimeUnit.SECONDS); + MessageExt msg = messageAtomic.get(); + assertThat(msg).isNotNull(); + assertThat(msg.getTopic()).isEqualTo(topic); + assertThat(msg.getBody()).isEqualTo(new byte[] {'a'}); } private PullRequest createPullRequest() { @@ -270,7 +283,7 @@ public class DefaultMQConsumerWithTraceTest { queueData.setPerm(6); queueData.setReadQueueNums(3); queueData.setWriteQueueNums(4); - queueData.setTopicSynFlag(0); + queueData.setTopicSysFlag(0); queueDataList.add(queueData); topicRouteData.setQueueDatas(queueDataList); return topicRouteData; @@ -306,7 +319,7 @@ public class DefaultMQConsumerWithTraceTest { queueData.setPerm(6); queueData.setReadQueueNums(1); queueData.setWriteQueueNums(1); - queueData.setTopicSynFlag(1); + queueData.setTopicSysFlag(1); queueDataList.add(queueData); topicRouteData.setQueueDatas(queueDataList); return topicRouteData; diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..67ae194b880ce5a497b7d0aaac72566276fb1ec5 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +import java.util.concurrent.ConcurrentMap; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.consumer.store.ReadOffsetType; +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQAdminImpl; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; +import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.impl.consumer.RebalanceImpl; +import org.apache.rocketmq.client.impl.consumer.RebalanceService; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.message.MessageClientExt; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.RPCHook; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultMQLitePullConsumerWithTraceTest { + + @Spy + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); + + @Mock + private MQClientAPIImpl mQClientAPIImpl; + @Mock + private MQAdminImpl mQAdminImpl; + + private AsyncTraceDispatcher asyncTraceDispatcher; + private DefaultMQProducer traceProducer; + private RebalanceImpl rebalanceImpl; + private OffsetStore offsetStore; + private DefaultLitePullConsumerImpl litePullConsumerImpl; + private String consumerGroup = "LitePullConsumerGroup"; + private String topic = "LitePullConsumerTest"; + private String brokerName = "BrokerA"; + private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); + + private String customerTraceTopic = "rmq_trace_topic_12345"; + + @Before + public void init() throws Exception { + Field field = MQClientInstance.class.getDeclaredField("rebalanceService"); + field.setAccessible(true); + RebalanceService rebalanceService = (RebalanceService) field.get(mQClientFactory); + field = RebalanceService.class.getDeclaredField("waitInterval"); + field.setAccessible(true); + field.set(rebalanceService, 100); + } + + @Test + public void testSubscribe_PollMessageSuccess_WithDefaultTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = createLitePullConsumerWithDefaultTraceTopic(); + try { + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testSubscribe_PollMessageSuccess_WithCustomizedTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = createLitePullConsumerWithCustomizedTraceTopic(); + try { + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + + private DefaultLitePullConsumer createLitePullConsumerWithDefaultTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setEnableMsgTrace(true); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.subscribe(topic, "*"); + suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; + } + + private DefaultLitePullConsumer createLitePullConsumerWithCustomizedTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setEnableMsgTrace(true); + litePullConsumer.setCustomizedTraceTopic(customerTraceTopic); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.subscribe(topic, "*"); + suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; + } + + private void initDefaultLitePullConsumer(DefaultLitePullConsumer litePullConsumer) throws Exception { + asyncTraceDispatcher = (AsyncTraceDispatcher) litePullConsumer.getTraceDispatcher(); + traceProducer = asyncTraceDispatcher.getTraceProducer(); + Field field = DefaultLitePullConsumer.class.getDeclaredField("defaultLitePullConsumerImpl"); + field.setAccessible(true); + litePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(litePullConsumerImpl, mQClientFactory); + + PullAPIWrapper pullAPIWrapper = litePullConsumerImpl.getPullAPIWrapper(); + field = PullAPIWrapper.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(pullAPIWrapper, mQClientFactory); + + Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + fieldTrace.setAccessible(true); + fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + field = MQClientInstance.class.getDeclaredField("mQAdminImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQAdminImpl); + + field = DefaultLitePullConsumerImpl.class.getDeclaredField("rebalanceImpl"); + field.setAccessible(true); + rebalanceImpl = (RebalanceImpl) field.get(litePullConsumerImpl); + field = RebalanceImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(rebalanceImpl, mQClientFactory); + + offsetStore = spy(litePullConsumerImpl.getOffsetStore()); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("offsetStore"); + field.setAccessible(true); + field.set(litePullConsumerImpl, offsetStore); + + traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); + + when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), + anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) + .thenAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + MessageClientExt messageClientExt = new MessageClientExt(); + messageClientExt.setTopic(topic); + messageClientExt.setQueueId(0); + messageClientExt.setMsgId("123"); + messageClientExt.setBody(new byte[] {'a'}); + messageClientExt.setOffsetMsgId("234"); + messageClientExt.setBornHost(new InetSocketAddress(8080)); + messageClientExt.setStoreHost(new InetSocketAddress(8080)); + PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); + return pullResult; + } + }); + + when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(new FindBrokerResult("127.0.0.1:10911", false)); + + doReturn(Collections.singletonList(mQClientFactory.getClientId())).when(mQClientFactory).findConsumerIdList(anyString(), anyString()); + + doReturn(123L).when(offsetStore).readOffset(any(MessageQueue.class), any(ReadOffsetType.class)); + + } + + private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, + List messageExtList) throws Exception { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (MessageExt messageExt : messageExtList) { + outputStream.write(MessageDecoder.encode(messageExt, false)); + } + return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); + } + + private MessageQueue createMessageQueue() { + MessageQueue messageQueue = new MessageQueue(); + messageQueue.setBrokerName(brokerName); + messageQueue.setQueueId(0); + messageQueue.setTopic(topic); + return messageQueue; + } + + private TopicRouteData createTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("BrokerA"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("BrokerA"); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("123"); + sendResult.setOffsetMsgId("123"); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + return sendResult; + } + + private static void suppressUpdateTopicRouteInfoFromNameServer(DefaultLitePullConsumer litePullConsumer) throws IllegalAccessException { + DefaultLitePullConsumerImpl defaultLitePullConsumerImpl = (DefaultLitePullConsumerImpl) FieldUtils.readDeclaredField(litePullConsumer, "defaultLitePullConsumerImpl", true); + if (litePullConsumer.getMessageModel() == MessageModel.CLUSTERING) { + litePullConsumer.changeInstanceNameToPID(); + } + MQClientInstance mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(litePullConsumer, (RPCHook) FieldUtils.readDeclaredField(defaultLitePullConsumerImpl, "rpcHook", true))); + ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); + factoryTable.put(litePullConsumer.buildMQClientId(), mQClientFactory); + doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); + } + +} \ No newline at end of file diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java new file mode 100644 index 0000000000000000000000000000000000000000..5d64a93f03987f80122a3e468e7af28a7046b374 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.trace; + +import io.opentracing.mock.MockSpan; +import io.opentracing.mock.MockTracer; +import io.opentracing.tag.Tags; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.client.trace.hook.SendMessageOpenTracingHookImpl; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageType; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultMQProducerWithOpenTracingTest { + + @Spy + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); + @Mock + private MQClientAPIImpl mQClientAPIImpl; + + private DefaultMQProducer producer; + + private Message message; + private String topic = "FooBar"; + private String producerGroupPrefix = "FooBar_PID"; + private String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); + private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); + private MockTracer tracer = new MockTracer(); + + @Before + public void init() throws Exception { + + producer = new DefaultMQProducer(producerGroupTemp); + producer.getDefaultMQProducerImpl().registerSendMessageHook( + new SendMessageOpenTracingHookImpl(tracer)); + producer.setNamesrvAddr("127.0.0.1:9876"); + message = new Message(topic, new byte[] {'a', 'b', 'c'}); + + producer.start(); + + Field field = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(producer.getDefaultMQProducerImpl(), mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl()); + + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(createSendResult(SendStatus.SEND_OK)); + + } + + @Test + public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, producer.getDefaultMQProducerImpl()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + producer.send(message); + assertThat(tracer.finishedSpans().size()).isEqualTo(1); + MockSpan span = tracer.finishedSpans().get(0); + assertThat(span.tags().get(Tags.MESSAGE_BUS_DESTINATION.getKey())).isEqualTo(topic); + assertThat(span.tags().get(Tags.SPAN_KIND.getKey())).isEqualTo(Tags.SPAN_KIND_PRODUCER); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_MSG_ID)).isEqualTo("123"); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_BODY_LENGTH)).isEqualTo(3); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_REGION_ID)).isEqualTo("HZ"); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_MSG_TYPE)).isEqualTo(MessageType.Normal_Msg.name()); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_SOTRE_HOST)).isEqualTo("127.0.0.1:10911"); + } + + @After + public void terminate() { + producer.shutdown(); + } + + public static TopicRouteData createTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("BrokerA"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("BrokerA"); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("123"); + sendResult.setOffsetMsgId("123"); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + return sendResult; + } + +} diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java index 3759acba139f9d7eb3d343855acbc27e338512b7..62b34175aa2dbf5fad30076a16ad2b1696addca2 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java @@ -31,12 +31,12 @@ import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.exception.RemotingException; import org.junit.After; import org.junit.Before; @@ -75,7 +75,7 @@ public class DefaultMQProducerWithTraceTest { private String topic = "FooBar"; private String producerGroupPrefix = "FooBar_PID"; private String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); - private String producerGroupTraceTemp = MixAll.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); + private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); private String customerTraceTopic = "rmq_trace_topic_12345"; @Before @@ -168,7 +168,7 @@ public class DefaultMQProducerWithTraceTest { queueData.setPerm(6); queueData.setReadQueueNums(3); queueData.setWriteQueueNums(4); - queueData.setTopicSynFlag(0); + queueData.setTopicSysFlag(0); queueDataList.add(queueData); topicRouteData.setQueueDatas(queueDataList); return topicRouteData; @@ -204,7 +204,7 @@ public class DefaultMQProducerWithTraceTest { queueData.setPerm(6); queueData.setReadQueueNums(1); queueData.setWriteQueueNums(1); - queueData.setTopicSynFlag(1); + queueData.setTopicSysFlag(1); queueDataList.add(queueData); topicRouteData.setQueueDatas(queueDataList); return topicRouteData; diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java new file mode 100644 index 0000000000000000000000000000000000000000..af5a7053957929e62e3d934299a3a6558742b5bf --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.trace; + +import org.apache.rocketmq.client.producer.LocalTransactionState; +import org.apache.rocketmq.common.message.MessageType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class TraceDataEncoderTest { + + private String traceData; + + private long time; + + @Before + public void init() { + time = System.currentTimeMillis(); + traceData = new StringBuilder() + .append("Pub").append(TraceConstants.CONTENT_SPLITOR) + .append(time).append(TraceConstants.CONTENT_SPLITOR) + .append("DefaultRegion").append(TraceConstants.CONTENT_SPLITOR) + .append("PID-test").append(TraceConstants.CONTENT_SPLITOR) + .append("topic-test").append(TraceConstants.CONTENT_SPLITOR) + .append("AC1415116D1418B4AAC217FE1B4E0000").append(TraceConstants.CONTENT_SPLITOR) + .append("Tags").append(TraceConstants.CONTENT_SPLITOR) + .append("Keys").append(TraceConstants.CONTENT_SPLITOR) + .append("127.0.0.1:10911").append(TraceConstants.CONTENT_SPLITOR) + .append(26).append(TraceConstants.CONTENT_SPLITOR) + .append(245).append(TraceConstants.CONTENT_SPLITOR) + .append(MessageType.Normal_Msg.ordinal()).append(TraceConstants.CONTENT_SPLITOR) + .append("0A9A002600002A9F0000000000002329").append(TraceConstants.CONTENT_SPLITOR) + .append(true).append(TraceConstants.FIELD_SPLITOR) + .toString(); + } + + @Test + public void testDecoderFromTraceDataString() { + List contexts = TraceDataEncoder.decoderFromTraceDataString(traceData); + Assert.assertEquals(contexts.size(), 1); + Assert.assertEquals(contexts.get(0).getTraceType(), TraceType.Pub); + } + + + @Test + public void testEncoderFromContextBean() { + TraceContext context = new TraceContext(); + context.setTraceType(TraceType.Pub); + context.setGroupName("PID-test"); + context.setRegionId("DefaultRegion"); + context.setCostTime(245); + context.setSuccess(true); + context.setTimeStamp(time); + TraceBean traceBean = new TraceBean(); + traceBean.setTopic("topic-test"); + traceBean.setKeys("Keys"); + traceBean.setTags("Tags"); + traceBean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000"); + traceBean.setOffsetMsgId("0A9A002600002A9F0000000000002329"); + traceBean.setStoreHost("127.0.0.1:10911"); + traceBean.setStoreTime(time); + traceBean.setMsgType(MessageType.Normal_Msg); + traceBean.setBodyLength(26); + List traceBeans = new ArrayList(); + traceBeans.add(traceBean); + context.setTraceBeans(traceBeans); + TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(context); + + Assert.assertEquals(traceTransferBean.getTransData(), traceData); + Assert.assertEquals(traceTransferBean.getTransKey().size(), 2); + } + + @Test + public void testEncoderFromContextBean_EndTransaction() { + TraceContext context = new TraceContext(); + context.setTraceType(TraceType.EndTransaction); + context.setGroupName("PID-test"); + context.setRegionId("DefaultRegion"); + context.setTimeStamp(time); + TraceBean traceBean = new TraceBean(); + traceBean.setTopic("topic-test"); + traceBean.setKeys("Keys"); + traceBean.setTags("Tags"); + traceBean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000"); + traceBean.setStoreHost("127.0.0.1:10911"); + traceBean.setMsgType(MessageType.Trans_msg_Commit); + traceBean.setTransactionId("transactionId"); + traceBean.setTransactionState(LocalTransactionState.COMMIT_MESSAGE); + traceBean.setFromTransactionCheck(false); + List traceBeans = new ArrayList(); + traceBeans.add(traceBean); + context.setTraceBeans(traceBeans); + TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(context); + + Assert.assertEquals(traceTransferBean.getTransKey().size(), 2); + String traceData = traceTransferBean.getTransData(); + TraceContext contextAfter = TraceDataEncoder.decoderFromTraceDataString(traceData).get(0); + Assert.assertEquals(context.getTraceType(), contextAfter.getTraceType()); + Assert.assertEquals(context.getTimeStamp(), contextAfter.getTimeStamp()); + Assert.assertEquals(context.getGroupName(), contextAfter.getGroupName()); + TraceBean before = context.getTraceBeans().get(0); + TraceBean after = contextAfter.getTraceBeans().get(0); + Assert.assertEquals(before.getTopic(), after.getTopic()); + Assert.assertEquals(before.getMsgId(), after.getMsgId()); + Assert.assertEquals(before.getTags(), after.getTags()); + Assert.assertEquals(before.getKeys(), after.getKeys()); + Assert.assertEquals(before.getStoreHost(), after.getStoreHost()); + Assert.assertEquals(before.getMsgType(), after.getMsgType()); + Assert.assertEquals(before.getClientHost(), after.getClientHost()); + Assert.assertEquals(before.getTransactionId(), after.getTransactionId()); + Assert.assertEquals(before.getTransactionState(), after.getTransactionState()); + Assert.assertEquals(before.isFromTransactionCheck(), after.isFromTransactionCheck()); + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java new file mode 100644 index 0000000000000000000000000000000000000000..b1fdbaf965ad1d6fe30e35ea9fdcc672a52b29e2 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.trace; + +import org.apache.commons.codec.Charsets; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageType; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class TraceViewTest { + + @Test + public void testDecodeFromTraceTransData() { + String messageBody = new StringBuilder() + .append("Pub").append(TraceConstants.CONTENT_SPLITOR) + .append(System.currentTimeMillis()).append(TraceConstants.CONTENT_SPLITOR) + .append("DefaultRegion").append(TraceConstants.CONTENT_SPLITOR) + .append("PID-test").append(TraceConstants.CONTENT_SPLITOR) + .append("topic-test").append(TraceConstants.CONTENT_SPLITOR) + .append("AC1415116D1418B4AAC217FE1B4E0000").append(TraceConstants.CONTENT_SPLITOR) + .append("Tags").append(TraceConstants.CONTENT_SPLITOR) + .append("Keys").append(TraceConstants.CONTENT_SPLITOR) + .append("127.0.0.1:10911").append(TraceConstants.CONTENT_SPLITOR) + .append(26).append(TraceConstants.CONTENT_SPLITOR) + .append(245).append(TraceConstants.CONTENT_SPLITOR) + .append(MessageType.Normal_Msg.ordinal()).append(TraceConstants.CONTENT_SPLITOR) + .append("0A9A002600002A9F0000000000002329").append(TraceConstants.CONTENT_SPLITOR) + .append(true).append(TraceConstants.FIELD_SPLITOR) + .toString(); + MessageExt message = new MessageExt(); + message.setBody(messageBody.getBytes(Charsets.UTF_8)); + String key = "AC1415116D1418B4AAC217FE1B4E0000"; + List traceViews = TraceView.decodeFromTraceTransData(key, message); + Assert.assertEquals(traceViews.size(), 1); + Assert.assertEquals(traceViews.get(0).getMsgId(), key); + + key = "AD4233434334AAC217FEFFD0000"; + traceViews = TraceView.decodeFromTraceTransData(key, message); + Assert.assertEquals(traceViews.size(), 0); + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java new file mode 100644 index 0000000000000000000000000000000000000000..dd6d1083ce03853f62eec696fdc256fa53aced7a --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.trace; + +import io.opentracing.mock.MockSpan; +import io.opentracing.mock.MockTracer; +import io.opentracing.tag.Tags; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.producer.LocalTransactionState; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.client.producer.TransactionListener; +import org.apache.rocketmq.client.producer.TransactionMQProducer; +import org.apache.rocketmq.client.trace.hook.EndTransactionOpenTracingHookImpl; +import org.apache.rocketmq.client.trace.hook.SendMessageOpenTracingHookImpl; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.message.MessageType; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TransactionMQProducerWithOpenTracingTest { + + @Spy + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); + @Mock + private MQClientAPIImpl mQClientAPIImpl; + + private TransactionMQProducer producer; + + private Message message; + private String topic = "FooBar"; + private String producerGroupPrefix = "FooBar_PID"; + private String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); + private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); + private MockTracer tracer = new MockTracer(); + @Before + public void init() throws Exception { + TransactionListener transactionListener = new TransactionListener() { + @Override + public LocalTransactionState executeLocalTransaction(Message msg, Object arg) { + return LocalTransactionState.COMMIT_MESSAGE; + } + + @Override + public LocalTransactionState checkLocalTransaction(MessageExt msg) { + return LocalTransactionState.COMMIT_MESSAGE; + } + }; + producer = new TransactionMQProducer(producerGroupTemp); + producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageOpenTracingHookImpl(tracer)); + producer.getDefaultMQProducerImpl().registerEndTransactionHook(new EndTransactionOpenTracingHookImpl(tracer)); + producer.setTransactionListener(transactionListener); + + producer.setNamesrvAddr("127.0.0.1:9876"); + message = new Message(topic, new byte[] {'a', 'b', 'c'}); + + producer.start(); + + Field field = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(producer.getDefaultMQProducerImpl(), mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl()); + + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(createSendResult(SendStatus.SEND_OK)); + + } + + @Test + public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, producer.getDefaultMQProducerImpl()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + producer.sendMessageInTransaction(message, null); + + assertThat(tracer.finishedSpans().size()).isEqualTo(2); + MockSpan span = tracer.finishedSpans().get(1); + assertThat(span.tags().get(Tags.MESSAGE_BUS_DESTINATION.getKey())).isEqualTo(topic); + assertThat(span.tags().get(Tags.SPAN_KIND.getKey())).isEqualTo(Tags.SPAN_KIND_PRODUCER); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_MSG_ID)).isEqualTo("123"); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_MSG_TYPE)).isEqualTo(MessageType.Trans_msg_Commit.name()); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_TRANSACTION_STATE)).isEqualTo(LocalTransactionState.COMMIT_MESSAGE.name()); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_IS_FROM_TRANSACTION_CHECK)).isEqualTo(false); + } + + @After + public void terminate() { + producer.shutdown(); + } + + public static TopicRouteData createTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("BrokerA"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("BrokerA"); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("123"); + sendResult.setOffsetMsgId(MessageDecoder.createMessageId(new InetSocketAddress("127.0.0.1", 12), 1)); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + sendResult.setMessageQueue(new MessageQueue(topic, "broker-trace", 0)); + return sendResult; + } + +} diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f838817bf9d999e402f6055669cd6fe42c9a16d3 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.trace; + +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.EndTransactionContext; +import org.apache.rocketmq.client.hook.EndTransactionHook; +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.LocalTransactionState; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.client.producer.TransactionListener; +import org.apache.rocketmq.client.producer.TransactionMQProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TransactionMQProducerWithTraceTest { + + @Spy + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); + @Mock + private MQClientAPIImpl mQClientAPIImpl; + @Mock + private EndTransactionHook endTransactionHook; + + private AsyncTraceDispatcher asyncTraceDispatcher; + + private TransactionMQProducer producer; + private DefaultMQProducer traceProducer; + + private Message message; + private String topic = "FooBar"; + private String producerGroupPrefix = "FooBar_PID"; + private String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); + private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); + private String customerTraceTopic = "rmq_trace_topic_12345"; + + @Before + public void init() throws Exception { + TransactionListener transactionListener = new TransactionListener() { + @Override + public LocalTransactionState executeLocalTransaction(Message msg, Object arg) { + return LocalTransactionState.COMMIT_MESSAGE; + } + + @Override + public LocalTransactionState checkLocalTransaction(MessageExt msg) { + return LocalTransactionState.COMMIT_MESSAGE; + } + }; + producer = new TransactionMQProducer(null, producerGroupTemp, null, true, null); + producer.setTransactionListener(transactionListener); + + producer.setNamesrvAddr("127.0.0.1:9876"); + message = new Message(topic, new byte[] {'a', 'b', 'c'}); + asyncTraceDispatcher = (AsyncTraceDispatcher) producer.getTraceDispatcher(); + traceProducer = asyncTraceDispatcher.getTraceProducer(); + + producer.start(); + + Field field = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(producer.getDefaultMQProducerImpl(), mQClientFactory); + + Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + fieldTrace.setAccessible(true); + fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl()); + + Field fieldHooks = DefaultMQProducerImpl.class.getDeclaredField("endTransactionHookList"); + fieldHooks.setAccessible(true); + Listhooks = new ArrayList<>(); + hooks.add(endTransactionHook); + fieldHooks.set(producer.getDefaultMQProducerImpl(), hooks); + + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(createSendResult(SendStatus.SEND_OK)); + + } + + @Test + public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + AtomicReference context = new AtomicReference<>(); + doAnswer(mock -> { + context.set(mock.getArgument(0)); + return null; + }).when(endTransactionHook).endTransaction(any()); + producer.sendMessageInTransaction(message, null); + + EndTransactionContext ctx = context.get(); + assertThat(ctx.getProducerGroup()).isEqualTo(producerGroupTemp); + assertThat(ctx.getMsgId()).isEqualTo("123"); + assertThat(ctx.isFromTransactionCheck()).isFalse(); + assertThat(new String(ctx.getMessage().getBody())).isEqualTo(new String(message.getBody())); + assertThat(ctx.getMessage().getTopic()).isEqualTo(topic); + } + + @After + public void terminate() { + producer.shutdown(); + } + + public static TopicRouteData createTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("BrokerA"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("BrokerA"); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("123"); + sendResult.setOffsetMsgId(MessageDecoder.createMessageId(new InetSocketAddress("127.0.0.1", 12), 1)); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + sendResult.setMessageQueue(new MessageQueue(topic, "broker-trace", 0)); + return sendResult; + } + +} diff --git a/client/src/test/resources/log4j2.xml b/client/src/test/resources/log4j2.xml new file mode 100644 index 0000000000000000000000000000000000000000..52cf2a88697c905acf266f6a65becd18e1ee359c --- /dev/null +++ b/client/src/test/resources/log4j2.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/common/pom.xml b/common/pom.xml index 56c150f877b7b402f8ae9a36fe0369577fe64038..ac1d086b4ccf486b848beaae1813b70f81077911 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -27,11 +27,6 @@ rocketmq-common rocketmq-common ${project.version} - - 1.6 - 1.6 - - ${project.groupId} diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index bfe8a2104db0b5a1781f066791f5b61f9fa19d12..f710cdb40307bbafc31f63046ba95966c52eb873 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -21,6 +21,7 @@ import java.net.UnknownHostException; import org.apache.rocketmq.common.annotation.ImportantField; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingUtil; @@ -52,14 +53,13 @@ public class BrokerConfig { private boolean autoCreateSubscriptionGroup = true; private String messageStorePlugIn = ""; @ImportantField - private String msgTraceTopicName = MixAll.RMQ_SYS_TRACE_TOPIC; + private String msgTraceTopicName = TopicValidator.RMQ_SYS_TRACE_TOPIC; @ImportantField private boolean traceTopicEnable = false; /** - * thread numbers for send message thread pool, since spin lock will be used by default since 4.0.x, the default - * value is 1. + * thread numbers for send message thread pool. */ - private int sendMessageThreadPoolNums = 1; //16 + Runtime.getRuntime().availableProcessors() * 4; + private int sendMessageThreadPoolNums = Math.min(Runtime.getRuntime().availableProcessors(), 4); private int pullMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2; private int processReplyMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2; private int queryMessageThreadPoolNums = 8 + Runtime.getRuntime().availableProcessors(); @@ -72,7 +72,8 @@ public class BrokerConfig { /** * Thread numbers for EndTransactionProcessor */ - private int endTransactionThreadPoolNums = 8 + Runtime.getRuntime().availableProcessors() * 2; + private int endTransactionThreadPoolNums = Math.max(8 + Runtime.getRuntime().availableProcessors() * 2, + sendMessageThreadPoolNums * 4); private int flushConsumerOffsetInterval = 1000 * 5; diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index baff127428146bd160ed4e609cf37790e2a9e612..48cd1582cc2abfb05d68c22ace352d8615009b7d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.common; public class MQVersion { - public static final int CURRENT_VERSION = Version.V4_7_0.ordinal(); + public static final int CURRENT_VERSION = Version.V4_9_0.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index de259c931b9c4fbbc0ff97c3fb3cf2ad766ee064..9d95ecb5ea4cfaa7d445318f079afcbb658654a9 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -55,8 +55,6 @@ public class MixAll { public static final String WS_DOMAIN_SUBGROUP = System.getProperty("rocketmq.namesrv.domain.subgroup", "nsaddr"); //http://jmenv.tbsite.net:8080/rocketmq/nsaddr //public static final String WS_ADDR = "http://" + WS_DOMAIN_NAME + ":8080/rocketmq/" + WS_DOMAIN_SUBGROUP; - public static final String AUTO_CREATE_TOPIC_KEY_TOPIC = "TBW102"; // Will be created at broker when isAutoCreateTopicEnable - public static final String BENCHMARK_TOPIC = "BenchmarkTest"; public static final String DEFAULT_PRODUCER_GROUP = "DEFAULT_PRODUCER"; public static final String DEFAULT_CONSUMER_GROUP = "DEFAULT_CONSUMER"; public static final String TOOLS_CONSUMER_GROUP = "TOOLS_CONSUMER"; @@ -65,8 +63,6 @@ public class MixAll { public static final String CLIENT_INNER_PRODUCER_GROUP = "CLIENT_INNER_PRODUCER"; public static final String SELF_TEST_PRODUCER_GROUP = "SELF_TEST_P_GROUP"; public static final String SELF_TEST_CONSUMER_GROUP = "SELF_TEST_C_GROUP"; - public static final String SELF_TEST_TOPIC = "SELF_TEST_TOPIC"; - public static final String OFFSET_MOVED_EVENT = "OFFSET_MOVED_EVENT"; public static final String ONS_HTTP_PROXY_GROUP = "CID_ONS-HTTP-PROXY"; public static final String CID_ONSAPI_PERMISSION_GROUP = "CID_ONSAPI_PERMISSION"; public static final String CID_ONSAPI_OWNER_GROUP = "CID_ONSAPI_OWNER"; @@ -80,14 +76,9 @@ public class MixAll { public static final String RETRY_GROUP_TOPIC_PREFIX = "%RETRY%"; public static final String DLQ_GROUP_TOPIC_PREFIX = "%DLQ%"; public static final String REPLY_TOPIC_POSTFIX = "REPLY_TOPIC"; - public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; public static final String UNIQUE_MSG_QUERY_FLAG = "_UNIQUE_KEY_QUERY"; public static final String DEFAULT_TRACE_REGION_ID = "DefaultRegion"; public static final String CONSUME_CONTEXT_TYPE = "ConsumeContextType"; - public static final String RMQ_SYS_TRANS_HALF_TOPIC = "RMQ_SYS_TRANS_HALF_TOPIC"; - public static final String RMQ_SYS_TRACE_TOPIC = "RMQ_SYS_TRACE_TOPIC"; - public static final String RMQ_SYS_TRANS_OP_HALF_TOPIC = "RMQ_SYS_TRANS_OP_HALF_TOPIC"; - public static final String TRANS_CHECK_MAX_TIME_TOPIC = "TRANS_CHECK_MAX_TIME_TOPIC"; public static final String CID_SYS_RMQ_TRANS = "CID_RMQ_SYS_TRANS"; public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml"; public static final String REPLY_MESSAGE_FLAG = "reply"; @@ -115,10 +106,6 @@ public class MixAll { return consumerGroup.startsWith(CID_RMQ_SYS_PREFIX); } - public static boolean isSystemTopic(final String topic) { - return topic.startsWith(SYSTEM_TOPIC_PREFIX); - } - public static String getDLQTopic(final String consumerGroup) { return DLQ_GROUP_TOPIC_PREFIX + consumerGroup; } diff --git a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java index 2aab5b79234c8621222b75e43426987c804e30aa..aa6920817f14efeb58a421b4eb33fe33078cadaf 100644 --- a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java @@ -200,14 +200,20 @@ public class UtilAll { } public static double getDiskPartitionSpaceUsedPercent(final String path) { - if (null == path || path.isEmpty()) + if (null == path || path.isEmpty()) { + log.error("Error when measuring disk space usage, path is null or empty, path : {}", path); return -1; + } + try { File file = new File(path); - if (!file.exists()) + if (!file.exists()) { + log.error("Error when measuring disk space usage, file doesn't exist on this path: {}", path); return -1; + } + long totalSpace = file.getTotalSpace(); @@ -218,6 +224,7 @@ public class UtilAll { return usedSpace / (double) totalSpace; } } catch (Exception e) { + log.error("Error when measuring disk space usage, got exception: :", e); return -1; } @@ -454,33 +461,9 @@ public class UtilAll { if (ip.length != 4) { throw new RuntimeException("illegal ipv4 bytes"); } - - if (ip[0] >= (byte) 1 && ip[0] <= (byte) 126) { - if (ip[1] == (byte) 1 && ip[2] == (byte) 1 && ip[3] == (byte) 1) { - return false; - } - if (ip[1] == (byte) 0 && ip[2] == (byte) 0 && ip[3] == (byte) 0) { - return false; - } - return true; - } else if (ip[0] >= (byte) 128 && ip[0] <= (byte) 191) { - if (ip[2] == (byte) 1 && ip[3] == (byte) 1) { - return false; - } - if (ip[2] == (byte) 0 && ip[3] == (byte) 0) { - return false; - } - return true; - } else if (ip[0] >= (byte) 192 && ip[0] <= (byte) 223) { - if (ip[3] == (byte) 1) { - return false; - } - if (ip[3] == (byte) 0) { - return false; - } - return true; - } - return false; + + InetAddressValidator validator = InetAddressValidator.getInstance(); + return validator.isValidInet4Address(ipToIPv4Str(ip)); } private static boolean ipV6Check(byte[] ip) { diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java index 200dec24813b33769fd2a8e502de492e60302855..1fe9aa106e9ab8692da9e0fb34b9ef06e0b43f57 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java @@ -20,10 +20,10 @@ public class PermName { public static final int PERM_PRIORITY = 0x1 << 3; public static final int PERM_READ = 0x1 << 2; public static final int PERM_WRITE = 0x1 << 1; - public static final int PERM_INHERIT = 0x1 << 0; + public static final int PERM_INHERIT = 0x1; public static String perm2String(final int perm) { - final StringBuffer sb = new StringBuffer("---"); + final StringBuilder sb = new StringBuilder("---"); if (isReadable(perm)) { sb.replace(0, 1, "R"); } diff --git a/common/src/main/java/org/apache/rocketmq/common/filter/FilterAPI.java b/common/src/main/java/org/apache/rocketmq/common/filter/FilterAPI.java index 9268a6ef45d0663b1c5a08b81efc93e6ef988741..c5b51b1aff57b17ee24d010230cfae64d0d9a0a3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/filter/FilterAPI.java +++ b/common/src/main/java/org/apache/rocketmq/common/filter/FilterAPI.java @@ -36,8 +36,7 @@ public class FilterAPI { return simple; } - public static SubscriptionData buildSubscriptionData(final String consumerGroup, String topic, - String subString) throws Exception { + public static SubscriptionData buildSubscriptionData(String topic, String subString) throws Exception { SubscriptionData subscriptionData = new SubscriptionData(); subscriptionData.setTopic(topic); subscriptionData.setSubString(subString); @@ -67,7 +66,7 @@ public class FilterAPI { public static SubscriptionData build(final String topic, final String subString, final String type) throws Exception { if (ExpressionType.TAG.equals(type) || type == null) { - return buildSubscriptionData(null, topic, subString); + return buildSubscriptionData(topic, subString); } if (subString == null || subString.length() < 1) { diff --git a/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java b/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java index e72fb826013606e0ec9f636f06f06d038a2cb098..f57df26d6fe4794da0f2da75d62dc559ce37e901 100644 --- a/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java +++ b/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java @@ -20,7 +20,7 @@ package org.apache.rocketmq.common.hook; import java.nio.ByteBuffer; public interface FilterCheckHook { - public String hookName(); + String hookName(); - public boolean isFilterMatched(final boolean isUnitMode, final ByteBuffer byteBuffer); + boolean isFilterMatched(final boolean isUnitMode, final ByteBuffer byteBuffer); } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java index d048dde2b92d3b60035f5764dfd27191417c0ef6..2936c18eac9d45ff6c34cfa394d150701ed8cee3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java @@ -42,6 +42,7 @@ public class MessageDecoder { public static final char NAME_VALUE_SEPARATOR = 1; public static final char PROPERTY_SEPARATOR = 2; public static final int PHY_POS_POSITION = 4 + 4 + 4 + 4 + 4 + 8; + public static final int QUEUE_OFFSET_POSITION = 4 + 4 + 4 + 4 + 4; public static final int SYSFLAG_POSITION = 4 + 4 + 4 + 4 + 4 + 8 + 8; // public static final int BODY_SIZE_POSITION = 4 // 1 TOTALSIZE // + 4 // 2 MAGICCODE @@ -104,7 +105,7 @@ public class MessageDecoder { * * @param byteBuffer msg commit log buffer. */ - public static Map decodeProperties(java.nio.ByteBuffer byteBuffer) { + public static Map decodeProperties(ByteBuffer byteBuffer) { int sysFlag = byteBuffer.getInt(SYSFLAG_POSITION); int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; int storehostAddressLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 8 : 20; @@ -140,15 +141,15 @@ public class MessageDecoder { return null; } - public static MessageExt decode(java.nio.ByteBuffer byteBuffer) { + public static MessageExt decode(ByteBuffer byteBuffer) { return decode(byteBuffer, true, true, false); } - public static MessageExt clientDecode(java.nio.ByteBuffer byteBuffer, final boolean readBody) { + public static MessageExt clientDecode(ByteBuffer byteBuffer, final boolean readBody) { return decode(byteBuffer, readBody, true, true); } - public static MessageExt decode(java.nio.ByteBuffer byteBuffer, final boolean readBody) { + public static MessageExt decode(ByteBuffer byteBuffer, final boolean readBody) { return decode(byteBuffer, readBody, true, false); } @@ -263,12 +264,12 @@ public class MessageDecoder { } public static MessageExt decode( - java.nio.ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody) { + ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody) { return decode(byteBuffer, readBody, deCompressBody, false); } public static MessageExt decode( - java.nio.ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody, final boolean isClient) { + ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody, final boolean isClient) { try { MessageExt msgExt; @@ -390,11 +391,11 @@ public class MessageDecoder { return null; } - public static List decodes(java.nio.ByteBuffer byteBuffer) { + public static List decodes(ByteBuffer byteBuffer) { return decodes(byteBuffer, true); } - public static List decodes(java.nio.ByteBuffer byteBuffer, final boolean readBody) { + public static List decodes(ByteBuffer byteBuffer, final boolean readBody) { List msgExts = new ArrayList(); while (byteBuffer.hasRemaining()) { MessageExt msgExt = clientDecode(byteBuffer, readBody); diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/NamespaceUtil.java b/common/src/main/java/org/apache/rocketmq/common/protocol/NamespaceUtil.java index afd537612267ca04068e3338d8e67ea98fb3835b..704eb90bd7a583029c6c1bb0bbece9d16272902c 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/NamespaceUtil.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/NamespaceUtil.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.common.protocol; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.topic.TopicValidator; public class NamespaceUtil { public static final char NAMESPACE_SEPARATOR = '%'; @@ -38,19 +39,19 @@ public class NamespaceUtil { return resourceWithNamespace; } - StringBuffer strBuffer = new StringBuffer(); + StringBuilder stringBuilder = new StringBuilder(); if (isRetryTopic(resourceWithNamespace)) { - strBuffer.append(MixAll.RETRY_GROUP_TOPIC_PREFIX); + stringBuilder.append(MixAll.RETRY_GROUP_TOPIC_PREFIX); } if (isDLQTopic(resourceWithNamespace)) { - strBuffer.append(MixAll.DLQ_GROUP_TOPIC_PREFIX); + stringBuilder.append(MixAll.DLQ_GROUP_TOPIC_PREFIX); } String resourceWithoutRetryAndDLQ = withOutRetryAndDLQ(resourceWithNamespace); int index = resourceWithoutRetryAndDLQ.indexOf(NAMESPACE_SEPARATOR); if (index > 0) { String resourceWithoutNamespace = resourceWithoutRetryAndDLQ.substring(index + 1); - return strBuffer.append(resourceWithoutNamespace).toString(); + return stringBuilder.append(resourceWithoutNamespace).toString(); } return resourceWithNamespace; @@ -90,17 +91,17 @@ public class NamespaceUtil { } String resourceWithoutRetryAndDLQ = withOutRetryAndDLQ(resourceWithOutNamespace); - StringBuffer strBuffer = new StringBuffer(); + StringBuilder stringBuilder = new StringBuilder(); if (isRetryTopic(resourceWithOutNamespace)) { - strBuffer.append(MixAll.RETRY_GROUP_TOPIC_PREFIX); + stringBuilder.append(MixAll.RETRY_GROUP_TOPIC_PREFIX); } if (isDLQTopic(resourceWithOutNamespace)) { - strBuffer.append(MixAll.DLQ_GROUP_TOPIC_PREFIX); + stringBuilder.append(MixAll.DLQ_GROUP_TOPIC_PREFIX); } - return strBuffer.append(namespace).append(NAMESPACE_SEPARATOR).append(resourceWithoutRetryAndDLQ).toString(); + return stringBuilder.append(namespace).append(NAMESPACE_SEPARATOR).append(resourceWithoutRetryAndDLQ).toString(); } @@ -155,11 +156,11 @@ public class NamespaceUtil { return false; } - if (MixAll.isSystemTopic(resource) || MixAll.isSysConsumerGroup(resource)) { + if (TopicValidator.isSystemTopic(resource) || MixAll.isSysConsumerGroup(resource)) { return true; } - return MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC.equals(resource); + return false; } public static boolean isRetryTopic(String resource) { diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java index dff9e2f35ce5d93478d4794816355b85048f3e29..6591d7780cd3df202424fff94109020f99f71739 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java @@ -25,6 +25,8 @@ public class DeleteSubscriptionGroupRequestHeader implements CommandCustomHeader @CFNotNull private String groupName; + private boolean removeOffset; + @Override public void checkFields() throws RemotingCommandException { } @@ -36,4 +38,12 @@ public class DeleteSubscriptionGroupRequestHeader implements CommandCustomHeader public void setGroupName(String groupName) { this.groupName = groupName; } + + public boolean isRemoveOffset() { + return removeOffset; + } + + public void setRemoveOffset(boolean removeOffset) { + this.removeOffset = removeOffset; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/QueueData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/QueueData.java index 72a21f384cbb88ebcb7a92e65bccc097a9463e8a..2dbb2902c9be43c8514929db1338e28495fde401 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/QueueData.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/QueueData.java @@ -25,7 +25,7 @@ public class QueueData implements Comparable { private int readQueueNums; private int writeQueueNums; private int perm; - private int topicSynFlag; + private int topicSysFlag; public int getReadQueueNums() { return readQueueNums; @@ -51,12 +51,12 @@ public class QueueData implements Comparable { this.perm = perm; } - public int getTopicSynFlag() { - return topicSynFlag; + public int getTopicSysFlag() { + return topicSysFlag; } - public void setTopicSynFlag(int topicSynFlag) { - this.topicSynFlag = topicSynFlag; + public void setTopicSysFlag(int topicSysFlag) { + this.topicSysFlag = topicSysFlag; } @Override @@ -67,7 +67,7 @@ public class QueueData implements Comparable { result = prime * result + perm; result = prime * result + readQueueNums; result = prime * result + writeQueueNums; - result = prime * result + topicSynFlag; + result = prime * result + topicSysFlag; return result; } @@ -91,7 +91,7 @@ public class QueueData implements Comparable { return false; if (writeQueueNums != other.writeQueueNums) return false; - if (topicSynFlag != other.topicSynFlag) + if (topicSysFlag != other.topicSysFlag) return false; return true; } @@ -99,7 +99,7 @@ public class QueueData implements Comparable { @Override public String toString() { return "QueueData [brokerName=" + brokerName + ", readQueueNums=" + readQueueNums - + ", writeQueueNums=" + writeQueueNums + ", perm=" + perm + ", topicSynFlag=" + topicSynFlag + + ", writeQueueNums=" + writeQueueNums + ", perm=" + perm + ", topicSysFlag=" + topicSysFlag + "]"; } diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/RTStatsItem.java b/common/src/main/java/org/apache/rocketmq/common/stats/RTStatsItem.java new file mode 100644 index 0000000000000000000000000000000000000000..102148cdc2e751ac592efc0e0be0888542435d44 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/stats/RTStatsItem.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.stats; + +import org.apache.rocketmq.logging.InternalLogger; + +import java.util.concurrent.ScheduledExecutorService; + +/** + * A StatItem for response time, the only difference between from StatsItem is it has a different log output. + */ +public class RTStatsItem extends StatsItem { + + public RTStatsItem(String statsName, String statsKey, ScheduledExecutorService scheduledExecutorService, InternalLogger log) { + super(statsName, statsKey, scheduledExecutorService, log); + } + + /** + * For Response Time stat Item, the print detail should be a little different, TPS and SUM makes no sense. + * And we give a name "AVGRT" rather than AVGPT for value getAvgpt() + */ + @Override + protected String statPrintDetail(StatsSnapshot ss) { + return String.format("TIMES: %d AVGRT: %.2f", ss.getTimes(), ss.getAvgpt()); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java index 6304ea2c4a37f5696f08e85614d003e363d4347a..b078551ad2ca006d2b2c5f84589bbb3e059d068f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java +++ b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItem.java @@ -55,13 +55,14 @@ public class StatsItem { double tps = 0; double avgpt = 0; long sum = 0; + long timesDiff = 0; if (!csList.isEmpty()) { CallSnapshot first = csList.getFirst(); CallSnapshot last = csList.getLast(); sum = last.getValue() - first.getValue(); tps = (sum * 1000.0d) / (last.getTimestamp() - first.getTimestamp()); - long timesDiff = last.getTimes() - first.getTimes(); + timesDiff = last.getTimes() - first.getTimes(); if (timesDiff > 0) { avgpt = (sum * 1.0d) / timesDiff; } @@ -70,6 +71,7 @@ public class StatsItem { statsSnapshot.setSum(sum); statsSnapshot.setTps(tps); statsSnapshot.setAvgpt(avgpt); + statsSnapshot.setTimes(timesDiff); } return statsSnapshot; @@ -191,32 +193,25 @@ public class StatsItem { public void printAtMinutes() { StatsSnapshot ss = computeStatsData(this.csListMinute); - log.info(String.format("[%s] [%s] Stats In One Minute, SUM: %d TPS: %.2f AVGPT: %.2f", - this.statsName, - this.statsKey, - ss.getSum(), - ss.getTps(), - ss.getAvgpt())); + log.info(String.format("[%s] [%s] Stats In One Minute, ", this.statsName, this.statsKey) + statPrintDetail(ss)); } public void printAtHour() { StatsSnapshot ss = computeStatsData(this.csListHour); - log.info(String.format("[%s] [%s] Stats In One Hour, SUM: %d TPS: %.2f AVGPT: %.2f", - this.statsName, - this.statsKey, - ss.getSum(), - ss.getTps(), - ss.getAvgpt())); + log.info(String.format("[%s] [%s] Stats In One Hour, ", this.statsName, this.statsKey) + statPrintDetail(ss)); + } public void printAtDay() { StatsSnapshot ss = computeStatsData(this.csListDay); - log.info(String.format("[%s] [%s] Stats In One Day, SUM: %d TPS: %.2f AVGPT: %.2f", - this.statsName, - this.statsKey, - ss.getSum(), - ss.getTps(), - ss.getAvgpt())); + log.info(String.format("[%s] [%s] Stats In One Day, ", this.statsName, this.statsKey) + statPrintDetail(ss)); + } + + protected String statPrintDetail(StatsSnapshot ss) { + return String.format("SUM: %d TPS: %.2f AVGPT: %.2f", + ss.getSum(), + ss.getTps(), + ss.getAvgpt()); } public AtomicLong getValue() { diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java index bcf9665959e0830ccc07ba4efe462f6765946455..a28d008d3e2d07dd2be3b1dcda20a0241d8b023c 100644 --- a/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java +++ b/common/src/main/java/org/apache/rocketmq/common/stats/StatsItemSet.java @@ -158,6 +158,12 @@ public class StatsItemSet { statsItem.getTimes().addAndGet(incTimes); } + public void addRTValue(final String statsKey, final int incValue, final int incTimes) { + StatsItem statsItem = this.getAndCreateRTStatsItem(statsKey); + statsItem.getValue().addAndGet(incValue); + statsItem.getTimes().addAndGet(incTimes); + } + public void delValue(final String statsKey) { StatsItem statsItem = this.statsItemTable.get(statsKey); if (null != statsItem) { @@ -196,9 +202,21 @@ public class StatsItemSet { } public StatsItem getAndCreateStatsItem(final String statsKey) { + return getAndCreateItem(statsKey, false); + } + + public StatsItem getAndCreateRTStatsItem(final String statsKey) { + return getAndCreateItem(statsKey, true); + } + + public StatsItem getAndCreateItem(final String statsKey, boolean rtItem) { StatsItem statsItem = this.statsItemTable.get(statsKey); if (null == statsItem) { - statsItem = new StatsItem(this.statsName, statsKey, this.scheduledExecutorService, this.log); + if (rtItem) { + statsItem = new RTStatsItem(this.statsName, statsKey, this.scheduledExecutorService, this.log); + } else { + statsItem = new StatsItem(this.statsName, statsKey, this.scheduledExecutorService, this.log); + } StatsItem prev = this.statsItemTable.putIfAbsent(statsKey, statsItem); if (null != prev) { diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/StatsSnapshot.java b/common/src/main/java/org/apache/rocketmq/common/stats/StatsSnapshot.java index 136f21a7c8e12c0b5470358791d0cd99a41fd4d7..0cecce99359a44eea571f97a87b90464c6233d83 100644 --- a/common/src/main/java/org/apache/rocketmq/common/stats/StatsSnapshot.java +++ b/common/src/main/java/org/apache/rocketmq/common/stats/StatsSnapshot.java @@ -20,6 +20,8 @@ package org.apache.rocketmq.common.stats; public class StatsSnapshot { private long sum; private double tps; + + private long times; private double avgpt; public long getSum() { @@ -45,4 +47,12 @@ public class StatsSnapshot { public void setAvgpt(double avgpt) { this.avgpt = avgpt; } + + public long getTimes() { + return times; + } + + public void setTimes(long times) { + this.times = times; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java new file mode 100644 index 0000000000000000000000000000000000000000..7b0a8394a11aab76f2e33c520a8fb41f68f8cf78 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.topic; + +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TopicValidator { + + public static final String AUTO_CREATE_TOPIC_KEY_TOPIC = "TBW102"; // Will be created at broker when isAutoCreateTopicEnable + public static final String RMQ_SYS_SCHEDULE_TOPIC = "SCHEDULE_TOPIC_XXXX"; + public static final String RMQ_SYS_BENCHMARK_TOPIC = "BenchmarkTest"; + public static final String RMQ_SYS_TRANS_HALF_TOPIC = "RMQ_SYS_TRANS_HALF_TOPIC"; + public static final String RMQ_SYS_TRACE_TOPIC = "RMQ_SYS_TRACE_TOPIC"; + public static final String RMQ_SYS_TRANS_OP_HALF_TOPIC = "RMQ_SYS_TRANS_OP_HALF_TOPIC"; + public static final String RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC = "TRANS_CHECK_MAX_TIME_TOPIC"; + public static final String RMQ_SYS_SELF_TEST_TOPIC = "SELF_TEST_TOPIC"; + public static final String RMQ_SYS_OFFSET_MOVED_EVENT = "OFFSET_MOVED_EVENT"; + + public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; + + private static final String VALID_PATTERN_STR = "^[%|a-zA-Z0-9_-]+$"; + private static final Pattern PATTERN = Pattern.compile(VALID_PATTERN_STR); + private static final int TOPIC_MAX_LENGTH = 127; + + private static final Set SYSTEM_TOPIC_SET = new HashSet(); + + /** + * Topics'set which client can not send msg! + */ + private static final Set NOT_ALLOWED_SEND_TOPIC_SET = new HashSet(); + + static { + SYSTEM_TOPIC_SET.add(AUTO_CREATE_TOPIC_KEY_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_SCHEDULE_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_BENCHMARK_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_HALF_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_TRACE_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_OP_HALF_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_SELF_TEST_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_OFFSET_MOVED_EVENT); + + NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_SCHEDULE_TOPIC); + } + + private static boolean regularExpressionMatcher(String origin, Pattern pattern) { + if (pattern == null) { + return true; + } + Matcher matcher = pattern.matcher(origin); + return matcher.matches(); + } + + public static boolean validateTopic(String topic, RemotingCommand response) { + + if (UtilAll.isBlank(topic)) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The specified topic is blank."); + return false; + } + + if (!regularExpressionMatcher(topic, PATTERN)) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The specified topic contains illegal characters, allowing only " + VALID_PATTERN_STR); + return false; + } + + if (topic.length() > TOPIC_MAX_LENGTH) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The specified topic is longer than topic max length."); + return false; + } + + return true; + } + + public static boolean isSystemTopic(String topic, RemotingCommand response) { + if (isSystemTopic(topic)) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The topic[" + topic + "] is conflict with system topic."); + return true; + } + return false; + } + + public static boolean isSystemTopic(String topic) { + return SYSTEM_TOPIC_SET.contains(topic) || topic.startsWith(SYSTEM_TOPIC_PREFIX); + } + + public static boolean isNotAllowedSendTopic(String topic) { + return NOT_ALLOWED_SEND_TOPIC_SET.contains(topic); + } + + public static boolean isNotAllowedSendTopic(String topic, RemotingCommand response) { + if (isNotAllowedSendTopic(topic)) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("Sending message to topic[" + topic + "] is forbidden."); + return true; + } + return false; + } + + public static void addSystemTopic(String systemTopic) { + SYSTEM_TOPIC_SET.add(systemTopic); + } + + public static Set getSystemTopicSet() { + return SYSTEM_TOPIC_SET; + } + + public static Set getNotAllowedSendTopicSet() { + return NOT_ALLOWED_SEND_TOPIC_SET; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/NameServerAddressUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/NameServerAddressUtils.java index 6aaf3a28d90ef2633e38b6bce10209b914f7de22..85dc95feb4c796374db0d1628c798fe7f4febc29 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/NameServerAddressUtils.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/NameServerAddressUtils.java @@ -19,8 +19,7 @@ import org.apache.rocketmq.common.MixAll; public class NameServerAddressUtils { public static final String INSTANCE_PREFIX = "MQ_INST_"; public static final String INSTANCE_REGEX = INSTANCE_PREFIX + "\\w+_\\w+"; - public static final String ENDPOINT_PREFIX = "http://"; - public static final Pattern NAMESRV_ENDPOINT_PATTERN = Pattern.compile("^" + ENDPOINT_PREFIX + ".*"); + public static final String ENDPOINT_PREFIX = "(\\w+://|)"; public static final Pattern INST_ENDPOINT_PATTERN = Pattern.compile("^" + ENDPOINT_PREFIX + INSTANCE_REGEX + "\\..*"); public static String getNameServerAddresses() { @@ -35,6 +34,13 @@ public class NameServerAddressUtils { if (StringUtils.isEmpty(endpoint)) { return null; } - return endpoint.substring(ENDPOINT_PREFIX.length(), endpoint.indexOf('.')); + return endpoint.substring(endpoint.lastIndexOf("/") + 1, endpoint.indexOf('.')); + } + + public static String getNameSrvAddrFromNamesrvEndpoint(String nameSrvEndpoint) { + if (StringUtils.isEmpty(nameSrvEndpoint)) { + return null; + } + return nameSrvEndpoint.substring(nameSrvEndpoint.lastIndexOf('/') + 1); } } diff --git a/common/src/test/java/org/apache/rocketmq/common/filter/FilterAPITest.java b/common/src/test/java/org/apache/rocketmq/common/filter/FilterAPITest.java index 73ab09e61c4cd8ac5b4524815506af316c340b80..5190f88f47b531051a3d65fa91b5165b9ffb13fa 100644 --- a/common/src/test/java/org/apache/rocketmq/common/filter/FilterAPITest.java +++ b/common/src/test/java/org/apache/rocketmq/common/filter/FilterAPITest.java @@ -33,7 +33,7 @@ public class FilterAPITest { @Test public void testBuildSubscriptionData() throws Exception { SubscriptionData subscriptionData = - FilterAPI.buildSubscriptionData(group, topic, subString); + FilterAPI.buildSubscriptionData(topic, subString); assertThat(subscriptionData.getTopic()).isEqualTo(topic); assertThat(subscriptionData.getSubString()).isEqualTo(subString); String[] tags = subString.split("\\|\\|"); diff --git a/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java b/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java index 6d8c20383a2b6195ffda2e473d522dd26b2e4093..0a17c36b57d074de63df3b740337ddd1a94cee94 100644 --- a/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java @@ -35,12 +35,14 @@ public class MessageClientIDSetterTest { assertThat(ipStr).isEqualTo(ipStrFromID); } + @Test public void testGetPidFromID() { - int pid = UtilAll.getPid(); + // Temporary fix on MacOS + short pid = (short) UtilAll.getPid(); String uniqID = MessageClientIDSetter.createUniqID(); - int pidFromID = MessageClientIDSetter.getPidFromID(uniqID); + short pidFromID = (short) MessageClientIDSetter.getPidFromID(uniqID); assertThat(pid).isEqualTo(pidFromID); } diff --git a/common/src/test/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataTest.java b/common/src/test/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataTest.java index 764225d6a888a89f1cea2d6e49bf0cc8c8fa31a0..5c7c6d1d1ffa1e103d116f35c8796f5ee2f1dc5b 100644 --- a/common/src/test/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataTest.java @@ -43,7 +43,7 @@ public class TopicRouteDataTest { queueData.setPerm(6); queueData.setReadQueueNums(8); queueData.setWriteQueueNums(8); - queueData.setTopicSynFlag(0); + queueData.setTopicSysFlag(0); List queueDataList = new ArrayList(); queueDataList.add(queueData); @@ -78,7 +78,7 @@ public class TopicRouteDataTest { queueData.setPerm(6); queueData.setReadQueueNums(8); queueData.setWriteQueueNums(8); - queueData.setTopicSynFlag(0); + queueData.setTopicSysFlag(0); List queueDataList = new ArrayList(); queueDataList.add(queueData); diff --git a/common/src/test/java/org/apache/rocketmq/common/stats/StatsItemSetTest.java b/common/src/test/java/org/apache/rocketmq/common/stats/StatsItemSetTest.java index 4b4a86765b99aa4bba2437c3e1f2eab7de27facf..5b4c5d823d3357f38fb98601b2d2fb61f3190ca5 100644 --- a/common/src/test/java/org/apache/rocketmq/common/stats/StatsItemSetTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/stats/StatsItemSetTest.java @@ -46,14 +46,17 @@ public class StatsItemSetTest { @Test public void test_statsOfFirstStatisticsCycle() throws InterruptedException { - final StatsItemSet statsItemSet = new StatsItemSet("topicTest", scheduler, null); + final String tpsStatKey = "tpsTest"; + final String rtStatKey = "rtTest"; + final StatsItemSet statsItemSet = new StatsItemSet(tpsStatKey, scheduler, null); executor = new ThreadPoolExecutor(10, 20, 10, TimeUnit.SECONDS, new ArrayBlockingQueue(100), new ThreadFactoryImpl("testMultiThread")); for (int i = 0; i < 10; i++) { executor.submit(new Runnable() { @Override public void run() { - statsItemSet.addValue("topicTest", 2, 1); + statsItemSet.addValue(tpsStatKey, 2, 1); + statsItemSet.addRTValue(rtStatKey, 2, 1); } }); } @@ -63,14 +66,33 @@ public class StatsItemSetTest { } Thread.sleep(1000); } - // simulate schedule task execution - statsItemSet.getStatsItem("topicTest").samplingInSeconds(); - statsItemSet.getStatsItem("topicTest").samplingInMinutes(); - statsItemSet.getStatsItem("topicTest").samplingInHour(); + // simulate schedule task execution , tps stat + { + statsItemSet.getStatsItem(tpsStatKey).samplingInSeconds(); + statsItemSet.getStatsItem(tpsStatKey).samplingInMinutes(); + statsItemSet.getStatsItem(tpsStatKey).samplingInHour(); + + assertEquals(20L, statsItemSet.getStatsDataInMinute(tpsStatKey).getSum()); + assertEquals(20L, statsItemSet.getStatsDataInHour(tpsStatKey).getSum()); + assertEquals(20L, statsItemSet.getStatsDataInDay(tpsStatKey).getSum()); + assertEquals(10L, statsItemSet.getStatsDataInDay(tpsStatKey).getTimes()); + assertEquals(10L, statsItemSet.getStatsDataInHour(tpsStatKey).getTimes()); + assertEquals(10L, statsItemSet.getStatsDataInDay(tpsStatKey).getTimes()); + } + + // simulate schedule task execution , rt stat + { + statsItemSet.getStatsItem(rtStatKey).samplingInSeconds(); + statsItemSet.getStatsItem(rtStatKey).samplingInMinutes(); + statsItemSet.getStatsItem(rtStatKey).samplingInHour(); - assertEquals(20L, statsItemSet.getStatsDataInMinute("topicTest").getSum()); - assertEquals(20L, statsItemSet.getStatsDataInHour("topicTest").getSum()); - assertEquals(20L, statsItemSet.getStatsDataInDay("topicTest").getSum()); + assertEquals(20L, statsItemSet.getStatsDataInMinute(rtStatKey).getSum()); + assertEquals(20L, statsItemSet.getStatsDataInHour(rtStatKey).getSum()); + assertEquals(20L, statsItemSet.getStatsDataInDay(rtStatKey).getSum()); + assertEquals(10L, statsItemSet.getStatsDataInDay(rtStatKey).getTimes()); + assertEquals(10L, statsItemSet.getStatsDataInHour(rtStatKey).getTimes()); + assertEquals(10L, statsItemSet.getStatsDataInDay(rtStatKey).getTimes()); + } } private AtomicLong test_unit() throws InterruptedException { diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicValidatorTest.java b/common/src/test/java/org/apache/rocketmq/common/topic/TopicValidatorTest.java similarity index 52% rename from broker/src/test/java/org/apache/rocketmq/broker/topic/TopicValidatorTest.java rename to common/src/test/java/org/apache/rocketmq/common/topic/TopicValidatorTest.java index 78be63fc56aaf57c870c44cb095272604e651a78..bb49417b00cc3acc1bf4974a8bf5b6a9c0e768b3 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicValidatorTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/topic/TopicValidatorTest.java @@ -13,11 +13,11 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ -package org.apache.rocketmq.broker.topic; + */ +package org.apache.rocketmq.common.topic; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.junit.Test; @@ -40,18 +40,11 @@ public class TopicValidatorTest { assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); assertThat(response.getRemark()).contains("The specified topic contains illegal characters"); - clearResponse(response); - res = TopicValidator.validateTopic(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC, response); - assertThat(res).isFalse(); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); - assertThat(response.getRemark()).contains("The specified topic is conflict with AUTO_CREATE_TOPIC_KEY_TOPIC."); - clearResponse(response); res = TopicValidator.validateTopic(generateString(128), response); assertThat(res).isFalse(); assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); assertThat(response.getRemark()).contains("The specified topic is longer than topic max length."); - } @Test @@ -64,6 +57,76 @@ public class TopicValidatorTest { assertThat(response.getRemark()).isEmpty(); } + @Test + public void testAddSystemTopic() { + String topic = "SYSTEM_TOPIC_TEST"; + TopicValidator.addSystemTopic(topic); + assertThat(TopicValidator.getSystemTopicSet()).contains(topic); + } + + @Test + public void testIsSystemTopic() { + boolean res; + for (String topic : TopicValidator.getSystemTopicSet()) { + res = TopicValidator.isSystemTopic(topic); + assertThat(res).isTrue(); + } + + String topic = TopicValidator.SYSTEM_TOPIC_PREFIX + "_test"; + res = TopicValidator.isSystemTopic(topic); + assertThat(res).isTrue(); + + topic = "test_not_system_topic"; + res = TopicValidator.isSystemTopic(topic); + assertThat(res).isFalse(); + } + + @Test + public void testIsSystemTopicWithResponse() { + RemotingCommand response = RemotingCommand.createResponseCommand(-1, ""); + boolean res; + for (String topic : TopicValidator.getSystemTopicSet()) { + res = TopicValidator.isSystemTopic(topic, response); + assertThat(res).isTrue(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).isEqualTo("The topic[" + topic + "] is conflict with system topic."); + } + + String topic = "test_not_system_topic"; + res = TopicValidator.isSystemTopic(topic, response); + assertThat(res).isFalse(); + } + + @Test + public void testIsNotAllowedSendTopic() { + boolean res; + for (String topic : TopicValidator.getNotAllowedSendTopicSet()) { + res = TopicValidator.isNotAllowedSendTopic(topic); + assertThat(res).isTrue(); + } + + String topic = "test_allowed_send_topic"; + res = TopicValidator.isNotAllowedSendTopic(topic); + assertThat(res).isFalse(); + } + + @Test + public void testIsNotAllowedSendTopicWithResponse() { + RemotingCommand response = RemotingCommand.createResponseCommand(-1, ""); + + boolean res; + for (String topic : TopicValidator.getNotAllowedSendTopicSet()) { + res = TopicValidator.isNotAllowedSendTopic(topic, response); + assertThat(res).isTrue(); + assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION); + assertThat(response.getRemark()).isEqualTo("Sending message to topic[" + topic + "] is forbidden."); + } + + String topic = "test_allowed_send_topic"; + res = TopicValidator.isNotAllowedSendTopic(topic, response); + assertThat(res).isFalse(); + } + private static void clearResponse(RemotingCommand response) { response.setCode(-1); response.setRemark(""); diff --git a/common/src/test/java/org/apache/rocketmq/common/utils/NameServerAddressUtilsTest.java b/common/src/test/java/org/apache/rocketmq/common/utils/NameServerAddressUtilsTest.java new file mode 100644 index 0000000000000000000000000000000000000000..38cffdba9dd0b75463a3614c545af6c53bc71c47 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/utils/NameServerAddressUtilsTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.utils; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class NameServerAddressUtilsTest { + + private static String endpoint1 = "http://127.0.0.1:9876"; + private static String endpoint2 = "127.0.0.1:9876"; + private static String endpoint3 + = "http://MQ_INST_123456789_BXXUzaee.xxx:80"; + private static String endpoint4 = "MQ_INST_123456789_BXXUzaee.xxx:80"; + + @Test + public void testValidateInstanceEndpoint() { + assertThat(NameServerAddressUtils.validateInstanceEndpoint(endpoint1)).isEqualTo(false); + assertThat(NameServerAddressUtils.validateInstanceEndpoint(endpoint2)).isEqualTo(false); + assertThat(NameServerAddressUtils.validateInstanceEndpoint(endpoint3)).isEqualTo(true); + assertThat(NameServerAddressUtils.validateInstanceEndpoint(endpoint4)).isEqualTo(true); + } + + @Test + public void testParseInstanceIdFromEndpoint() { + assertThat(NameServerAddressUtils.parseInstanceIdFromEndpoint(endpoint3)).isEqualTo( + "MQ_INST_123456789_BXXUzaee"); + assertThat(NameServerAddressUtils.parseInstanceIdFromEndpoint(endpoint4)).isEqualTo( + "MQ_INST_123456789_BXXUzaee"); + } + + @Test + public void testGetNameSrvAddrFromNamesrvEndpoint() { + assertThat(NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(endpoint1)) + .isEqualTo("127.0.0.1:9876"); + assertThat(NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(endpoint2)) + .isEqualTo("127.0.0.1:9876"); + assertThat(NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(endpoint3)) + .isEqualTo("MQ_INST_123456789_BXXUzaee.xxx:80"); + assertThat(NameServerAddressUtils.getNameSrvAddrFromNamesrvEndpoint(endpoint4)) + .isEqualTo("MQ_INST_123456789_BXXUzaee.xxx:80"); + } +} diff --git a/distribution/LICENSE-BIN b/distribution/LICENSE-BIN index 372617233b33f51acef6aa9eebf0af63cc423147..bd431bfdea95d8ca8afeaffc1848481e03f82536 100644 --- a/distribution/LICENSE-BIN +++ b/distribution/LICENSE-BIN @@ -15,7 +15,7 @@ "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, - "control" means (properties) the power, direct or indirect, to cause the + "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. diff --git a/distribution/NOTICE-BIN b/distribution/NOTICE-BIN index c2f511fafe6decee5b827d47bbb52ac512fb8474..b7bba8daab0e740cc1fdfccb5d668f314f0da223 100644 --- a/distribution/NOTICE-BIN +++ b/distribution/NOTICE-BIN @@ -1,5 +1,5 @@ Apache RocketMQ -Copyright 2016-2019 The Apache Software Foundation +Copyright 2016-2021 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/distribution/benchmark/batchproducer.sh b/distribution/benchmark/batchproducer.sh new file mode 100644 index 0000000000000000000000000000000000000000..07297004e6260204f2d69e42f1ec728a9acd9401 --- /dev/null +++ b/distribution/benchmark/batchproducer.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sh ./runclass.sh org.apache.rocketmq.example.benchmark.BatchProducer $@ & diff --git a/distribution/benchmark/runclass.sh b/distribution/benchmark/runclass.sh index 339e11a2e4351945cb6b10c92bf2aa26dca39e67..12802ddf90d36221a26447a35219fc75a394778f 100644 --- a/distribution/benchmark/runclass.sh +++ b/distribution/benchmark/runclass.sh @@ -56,9 +56,9 @@ JAVA_OPT="${JAVA_OPT} -server -Xms1g -Xmx1g -Xmn256m -XX:PermSize=128m -XX:MaxPe JAVA_OPT="${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:+DisableExplicitGC" JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${GC_LOG_DIR}/rmq_run_class_gc_%p_%t.log -XX:+PrintGCDetails" JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" -JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${BASE_DIR}/lib" JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages" JAVA_OPT="${JAVA_OPT} -XX:+PerfDisableSharedMem" +JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${BASE_DIR}/lib:${JAVA_HOME}/lib/ext" #JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n" JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" diff --git a/distribution/benchmark/shutdown.sh b/distribution/benchmark/shutdown.sh new file mode 100644 index 0000000000000000000000000000000000000000..9ecd32600e3c08bc1743708f4edd83c4c2a9ce0e --- /dev/null +++ b/distribution/benchmark/shutdown.sh @@ -0,0 +1,63 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +case $1 in + producer) + + pid=`ps ax | grep -i 'org.apache.rocketmq.example.benchmark.Producer' |grep java | grep -v grep | awk '{print $1}'` + if [ -z "$pid" ] ; then + echo "No benchmark producer running." + exit -1; + fi + + echo "The benchmkar producer(${pid}) is running..." + + kill ${pid} + + echo "Send shutdown request to benchmark producer(${pid}) OK" + ;; + consumer) + + pid=`ps ax | grep -i 'org.apache.rocketmq.example.benchmark.Consumer' |grep java | grep -v grep | awk '{print $1}'` + if [ -z "$pid" ] ; then + echo "No benchmark consumer running." + exit -1; + fi + + echo "The benchmark consumer(${pid}) is running..." + + kill ${pid} + + echo "Send shutdown request to benchmark consumer(${pid}) OK" + ;; + tproducer) + + pid=`ps ax | grep -i 'org.apache.rocketmq.example.benchmark.TransactionProducer' |grep java | grep -v grep | awk '{print $1}'` + if [ -z "$pid" ] ; then + echo "No benchmark transaction producer running." + exit -1; + fi + + echo "The benchmkar transaction producer(${pid}) is running..." + + kill ${pid} + + echo "Send shutdown request to benchmark transaction producer(${pid}) OK" + ;; + *) + echo "Useage: shutdown producer | consumer | tproducer" +esac diff --git a/distribution/benchmark/tproducer.sh b/distribution/benchmark/tproducer.sh index ac4bbf3ee6dc18580adb75189ace3f70970213cd..9ced41f8324e82da5f5a17057591b3397e16be5b 100644 --- a/distribution/benchmark/tproducer.sh +++ b/distribution/benchmark/tproducer.sh @@ -15,4 +15,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -sh ./runclass.sh org.apache.rocketmq.example.benchmark.TransactionProducer $@ +sh ./runclass.sh org.apache.rocketmq.example.benchmark.TransactionProducer $@ & diff --git a/distribution/bin/dledger/fast-try.sh b/distribution/bin/dledger/fast-try.sh index ff8a96036d1a4b759fd72ae5979cf56cbe9f8534..acdde71b8df6ed63669bafb6de5ee48800a71c8f 100644 --- a/distribution/bin/dledger/fast-try.sh +++ b/distribution/bin/dledger/fast-try.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -20,25 +20,25 @@ CURRENT_DIR="$(cd "$(dirname "$0")"; pwd)" RMQ_DIR=$CURRENT_DIR/../.. cd $RMQ_DIR -function startNameserver() { +startNameserver() { export JAVA_OPT_EXT=" -Xms512m -Xmx512m " nohup bin/mqnamesrv & } -function startBroker() { +startBroker() { export JAVA_OPT_EXT=" -Xms1g -Xmx1g " conf_name=$1 nohup bin/mqbroker -c $conf_name & } -function stopNameserver() { +stopNameserver() { PIDS=$(ps -ef|grep java|grep NamesrvStartup|grep -v grep|awk '{print $2}') if [ ! -z "$PIDS" ]; then kill -s TERM $PIDS fi } -function stopBroker() { +stopBroker() { conf_name=$1 PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}') i=1 @@ -46,7 +46,7 @@ function stopBroker() { do echo "Waiting to kill ..." kill -s TERM $PIDS - ((i=$i+1)) + i=`expr $i + 1` sleep 2 PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}') done @@ -56,7 +56,7 @@ function stopBroker() { fi } -function stopAll() { +stopAll() { ps -ef|grep java|grep BrokerStartup|grep -v grep|awk '{print $2}'|xargs kill stopNameserver stopBroker ./conf/dledger/broker-n0.conf @@ -64,18 +64,18 @@ function stopAll() { stopBroker ./conf/dledger/broker-n2.conf } -function startAll() { +startAll() { startNameserver startBroker ./conf/dledger/broker-n0.conf startBroker ./conf/dledger/broker-n1.conf startBroker ./conf/dledger/broker-n2.conf } -function checkConf() { +checkConf() { if [ ! -f ./conf/dledger/broker-n0.conf -o ! -f ./conf/dledger/broker-n1.conf -o ! -f ./conf/dledger/broker-n2.conf ]; then echo "Make sure the ./conf/dledger/broker-n0.conf, ./conf/dledger/broker-n1.conf, ./conf/dledger/broker-n2.conf exists" - exit -1 - fi + exit 1 + fi } @@ -83,7 +83,7 @@ function checkConf() { ## Main if [ $# -lt 1 ]; then echo "Usage: sh $0 start|stop" - exit -1 + exit 1 fi action=$1 checkConf diff --git a/distribution/bin/mqadmin b/distribution/bin/mqadmin index cd0253bc87d55801bc35a4c9aa83c58beea9a5e8..980d6e3b6bda4ac9d0867a10902bcff6e0e1bb57 100644 --- a/distribution/bin/mqadmin +++ b/distribution/bin/mqadmin @@ -42,4 +42,4 @@ fi export ROCKETMQ_HOME -sh ${ROCKETMQ_HOME}/bin/tools.sh org.apache.rocketmq.tools.command.MQAdminStartup $@ +sh ${ROCKETMQ_HOME}/bin/tools.sh org.apache.rocketmq.tools.command.MQAdminStartup "$@" diff --git a/distribution/bin/runbroker.cmd b/distribution/bin/runbroker.cmd index eab7e30f9d4522501a645684ba89249f6c47b920..753a11e9a48eb592301ed599509233624e2c7ba8 100644 --- a/distribution/bin/runbroker.cmd +++ b/distribution/bin/runbroker.cmd @@ -36,7 +36,7 @@ set "JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow" set "JAVA_OPT=%JAVA_OPT% -XX:+AlwaysPreTouch" set "JAVA_OPT=%JAVA_OPT% -XX:MaxDirectMemorySize=15g" set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages -XX:-UseBiasedLocking" -set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib" +set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib;%JAVA_HOME%\jre\lib\ext" set "JAVA_OPT=%JAVA_OPT% -cp %CLASSPATH%" "%JAVA%" %JAVA_OPT% %* \ No newline at end of file diff --git a/distribution/bin/runbroker.sh b/distribution/bin/runbroker.sh index 0ccbf7725c229fd1f9300c98b19ce00884081655..1d1000ee4ce43e3da8869f7261fbd82515492871 100644 --- a/distribution/bin/runbroker.sh +++ b/distribution/bin/runbroker.sh @@ -72,7 +72,7 @@ JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=15g" JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" -JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${BASE_DIR}/lib" +JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${BASE_DIR}/lib:${JAVA_HOME}/lib/ext" #JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n" JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}" JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" diff --git a/distribution/bin/runserver.cmd b/distribution/bin/runserver.cmd index 48e32bf2d7cb81eaeccfa5040b168bbd1af3847c..76865f7a160fc6d720019dbad916438fcd993660 100644 --- a/distribution/bin/runserver.cmd +++ b/distribution/bin/runserver.cmd @@ -31,7 +31,7 @@ set "JAVA_OPT=%JAVA_OPT% -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollect set "JAVA_OPT=%JAVA_OPT% -verbose:gc -Xloggc:"%USERPROFILE%\rmq_srv_gc.log" -XX:+PrintGCDetails" set "JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow" set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages" -set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib" +set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib;%JAVA_HOME%\jre\lib\ext" set "JAVA_OPT=%JAVA_OPT% -cp "%CLASSPATH%"" "%JAVA%" %JAVA_OPT% %* \ No newline at end of file diff --git a/distribution/bin/runserver.sh b/distribution/bin/runserver.sh index d2bbdedbceafc8164a99509e007d829dad41a506..59f322085dfc89e155c479b2cb6b0e2e671d9a4a 100644 --- a/distribution/bin/runserver.sh +++ b/distribution/bin/runserver.sh @@ -62,15 +62,28 @@ choose_gc_log_directory() esac } +choose_gc_options() +{ + # Example of JAVA_MAJOR_VERSION value : '1', '9', '10', '11', ... + # '1' means releases befor Java 9 + JAVA_MAJOR_VERSION=$("$JAVA" -version 2>&1 | sed -r -n 's/.* version "([0-9]*).*$/\1/p') + if [ -z "$JAVA_MAJOR_VERSION" ] || [ "$JAVA_MAJOR_VERSION" -lt "9" ] ; then + JAVA_OPT="${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:-UseParNewGC" + JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${GC_LOG_DIR}/rmq_srv_gc_%p_%t.log -XX:+PrintGCDetails" + JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" + else + JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0" + JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${GC_LOG_DIR}/rmq_srv_gc_%p_%t.log:time,tags:filecount=5,filesize=30M" + fi +} + choose_gc_log_directory JAVA_OPT="${JAVA_OPT} -server -Xms4g -Xmx4g -Xmn2g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m" -JAVA_OPT="${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:-UseParNewGC" -JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${GC_LOG_DIR}/rmq_srv_gc_%p_%t.log -XX:+PrintGCDetails" -JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" +choose_gc_options JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages" -JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${BASE_DIR}/lib" +JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${BASE_DIR}/lib:${JAVA_HOME}/lib/ext" #JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n" JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}" JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" diff --git a/distribution/bin/tools.sh b/distribution/bin/tools.sh index 071794c63056a242f71b2915986414dc37c233a1..a13dc7e620aa3717cb8ea31a8ee06f28e07d1123 100644 --- a/distribution/bin/tools.sh +++ b/distribution/bin/tools.sh @@ -40,4 +40,4 @@ JAVA_OPT="${JAVA_OPT} -server -Xms1g -Xmx1g -Xmn256m -XX:MetaspaceSize=128m -XX: JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${BASE_DIR}/lib:${JAVA_HOME}/jre/lib/ext:${JAVA_HOME}/lib/ext" JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" -$JAVA ${JAVA_OPT} $@ +$JAVA ${JAVA_OPT} "$@" diff --git a/distribution/pom.xml b/distribution/pom.xml index 09e7ad03063b5e6566bda3651e929803ee8ef9ab..a5698bca1c614dfafce9590993a9923e265d346e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/distribution/release.xml b/distribution/release.xml index 64cffc81ea6ad51c0c28e73bc9d1bb3dfa9d1a2f..fd6e3db9427b5aeb96dc9077777f9c3196498011 100644 --- a/distribution/release.xml +++ b/distribution/release.xml @@ -74,6 +74,10 @@ lib/ + + io.jaegertracing:jaeger-core + io.jaegertracing:jaeger-client + diff --git a/docs/cn/Configuration_System.md b/docs/cn/Configuration_System.md new file mode 100644 index 0000000000000000000000000000000000000000..d72ef938136369fff5a34b2a010c4f6a6c495165 --- /dev/null +++ b/docs/cn/Configuration_System.md @@ -0,0 +1,70 @@ +# 系统配置 + +本节重点介绍系统（JVM/OS）的配置 +--- + +## **1 JVM 选项** ## + +建议使用最新发布的 JDK 1.8 版本。设置相同的 Xms 和 Xmx 值以防止 JVM 调整堆大小，并获得更好的性能。一种通用的JVM配置如下： + + -server -Xms8g -Xmx8g -Xmn4g + +设置 Direct ByteBuffer 内存大小。当 Direct ByteBuffer 达到指定大小时，将触发 Full GC： + + -XXMaxDirectMemorySize=15g + +如果你不在乎 RocketMQ broker 的启动时间，建议启用预分配 Java 堆以确保在 JVM 初始化期间为每个页面分配内存。你可以通过以下方式启用它： + + -XX+AlwaysPreTouch + +禁用偏向锁定可以减少 JVM 停顿： + + -XX-UseBiasedLocking + +关于垃圾收集器，推荐使用 JDK 1.8 的 G1 收集器： + + -XX+UseG1GC -XXG1HeapRegionSize=16m + -XXG1ReservePercent=25 + -XXInitiatingHeapOccupancyPercent=30 + +这些 GC 选项看起来有点激进，但事实证明它在生产环境中具有良好的性能 + +不要把-XXMaxGCPauseMillis 的值设置太小，否则JVM会使用一个小的新生代来实现这个目标，从而导致频繁发生minor GC。因此，建议使用滚动 GC 日志文件： + + -XX+UseGCLogFileRotation + -XXNumberOfGCLogFiles=5 + -XXGCLogFileSize=30m + +写 GC 文件会增加 broker 的延迟，因此可以考虑将 GC 日志文件重定向到内存文件系统： + + -Xloggcdevshmmq_gc_%p.log123 + +## 2 Linux 内核参数 ## + +在 bin 文件夹里，有一个 os.sh 脚本，里面列出了许多的内核参数，只需稍作更改即可用于生产用途。需特别关注以下参数，如想了解更多细节，请参考文档/proc/sys/vm/*。 + + + +- **vm.extra_free_kbytes**, 控制VM在后台回收（kswapd）开始的阈值和直接回收（通过分配进程）开始的阈值之间保留额外的空闲内存。通过使用这个参数，RocketMQ 可以避免在内存分配过程中出现高延迟。（与内核版本版本有关） + + + +- **vm.min_free_kbytes**, 该值不应设置低于1024KB，否则系统将遭到破坏，并且在高负载环境下容易出现死锁。 + + + + + +- **vm.max_map_count**, 规定进程可以拥有的最大内存映射区域数。 RocketMQ 使用 mmap 来加载 CommitLog 和 ConsumeQueue，因此建议将此参数设置为较大的值。 + + + +- **vm.swappiness**, 定义内核交换内存页的频率。该值若较大，则会导致频繁交换，较小则会减少交换量。为了避免交换延迟，建议将此值设为 10。 + + + +- **File descriptor limits**, RocketMQ 需要给文件（CommitLog 和 ConsumeQueue）和网络连接分配文件描述符。因此建议将该值设置为 655350。 + + + +- **Disk scheduler**, 推荐使用deadline IO 调度器，它可以为请求提供有保证的延迟。 \ No newline at end of file diff --git a/docs/cn/README b/docs/cn/README deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/docs/cn/RocketMQ_Example.md b/docs/cn/RocketMQ_Example.md index 0cf977ed7d8ef2b9cb6b0f9f3b870ee04baededf..d924ce1d7d1f54e8c68ed63ddaa040ad5445f13d 100644 --- a/docs/cn/RocketMQ_Example.md +++ b/docs/cn/RocketMQ_Example.md @@ -1,4 +1,43 @@ # 样例 +----- + * [目录](#样例) + * [1 基本样例](#1-基本样例) + * [1.1 加入依赖：](#11-加入依赖) + * [1.2 消息发送](#12-消息发送) + * [1、Producer端发送同步消息](#1producer端发送同步消息) + * [2、发送异步消息](#2发送异步消息) + * [3、单向发送消息](#3单向发送消息) + * [1.3 消费消息](#13-消费消息) + * [2 顺序消息样例](#2-顺序消息样例) + * [2.1 顺序消息生产](#21-顺序消息生产) + * [2.2 顺序消费消息](#22-顺序消费消息) + * [3 延时消息样例](#3-延时消息样例) + * [3.1 启动消费者等待传入订阅消息](#31-启动消费者等待传入订阅消息) + * [3.2 发送延时消息](#32-发送延时消息) + * [3.3 验证](#33-验证) + * [3.4 延时消息的使用场景](#34-延时消息的使用场景) + * [3.5 延时消息的使用限制](#35-延时消息的使用限制) + * [4 批量消息样例](#4-批量消息样例) + * [4.1 发送批量消息](#41-发送批量消息) + * [4.2 消息列表分割](#42-消息列表分割) + * [5 过滤消息样例](#5-过滤消息样例) + * [5.1 基本语法](#51-基本语法) + * [5.2 使用样例](#52-使用样例) + * [1、生产者样例](#1生产者样例) + * [2、消费者样例](#2消费者样例) + * [6 消息事务样例](#6-消息事务样例) + * [6.1 发送事务消息样例](#61-发送事务消息样例) + * [1、创建事务性生产者](#1创建事务性生产者) + * [2、实现事务的监听接口](#2实现事务的监听接口) + * [6.2 事务消息使用上的限制](#62-事务消息使用上的限制) + * [7 Logappender样例](#7-logappender样例) + * [7.1 log4j样例](#71-log4j样例) + * [7.2 log4j2样例](#72-log4j2样例) + * [7.3 logback样例](#73-logback样例) + * [8 OpenMessaging样例](#8-openmessaging样例) + * [8.1 OMSProducer样例](#81-omsproducer样例) + * [8.2 OMSPullConsumer](#82-omspullconsumer) + * [8.3 OMSPushConsumer](#83-omspushconsumer) ----- ## 1 基本样例 @@ -323,13 +362,6 @@ public class Producer { ### 2.2 顺序消费消息 ```java -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; -import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; -import org.apache.rocketmq.common.message.MessageExt; -import java.util.List; - package org.apache.rocketmq.example.order2; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; @@ -414,7 +446,7 @@ public class ScheduledMessageConsumer { public ConsumeConcurrentlyStatus consumeMessage(List messages, ConsumeConcurrentlyContext context) { for (MessageExt message : messages) { // Print approximate delay time period - System.out.println("Receive message[msgId=" + message.getMsgId() + "] " + (System.currentTimeMillis() - message.getStoreTimestamp()) + "ms later"); + System.out.println("Receive message[msgId=" + message.getMsgId() + "] " + (System.currentTimeMillis() - message.getBornTimestamp()) + "ms later"); } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } diff --git a/docs/cn/architecture.md b/docs/cn/architecture.md index 7b41edaea779a2c39924a59d00c4c8b693f881bf..42df9f07b2938187ca478e27ee789ff7f5fa240b 100644 --- a/docs/cn/architecture.md +++ b/docs/cn/architecture.md @@ -12,11 +12,11 @@ RocketMQ架构上主要分为四部分，如上图所示: - NameServer：NameServer是一个非常简单的Topic路由注册中心，其角色类似Dubbo中的zookeeper，支持Broker的动态注册与发现。主要包括两个功能：Broker管理，NameServer接受Broker集群的注册信息并且保存下来作为路由信息的基本数据。然后提供心跳检测机制，检查Broker是否还存活；路由信息管理，每个NameServer将保存关于Broker集群的整个路由信息和用于客户端查询的队列信息。然后Producer和Conumser通过NameServer就可以知道整个Broker集群的路由信息，从而进行消息的投递和消费。NameServer通常也是集群的方式部署，各实例间相互不进行信息通讯。Broker是向每一台NameServer注册自己的路由信息，所以每一个NameServer实例上面都保存一份完整的路由信息。当某个NameServer因某种原因下线了，Broker仍然可以向其它NameServer同步其路由信息，Producer,Consumer仍然可以动态感知Broker的路由的信息。 - BrokerServer：Broker主要负责消息的存储、投递和查询以及服务高可用保证，为了实现这些功能，Broker包含了以下几个重要子模块。 -1. Remoting Module：整个Broker的实体，负责处理来自clients端的请求。 -2. Client Manager：负责管理客户端(Producer/Consumer)和维护Consumer的Topic订阅信息 -3. Store Service：提供方便简单的API接口处理消息存储到物理硬盘和查询功能。 -4. HA Service：高可用服务，提供Master Broker 和 Slave Broker之间的数据同步功能。 -5. Index Service：根据特定的Message key对投递到Broker的消息进行索引服务，以提供消息的快速查询。 + 1. Remoting Module：整个Broker的实体，负责处理来自clients端的请求。 + 2. Client Manager：负责管理客户端(Producer/Consumer)和维护Consumer的Topic订阅信息 + 3. Store Service：提供方便简单的API接口处理消息存储到物理硬盘和查询功能。 + 4. HA Service：高可用服务，提供Master Broker 和 Slave Broker之间的数据同步功能。 + 5. Index Service：根据特定的Message key对投递到Broker的消息进行索引服务，以提供消息的快速查询。 ![](image/rocketmq_architecture_2.png) diff --git a/docs/cn/best_practice.md b/docs/cn/best_practice.md index e4b04e0d564c4d4b8c0d494df3c1a707f6de57a4..4ad6bb23fe7332bfb87d4eeb42997755f58c9777 100755 --- a/docs/cn/best_practice.md +++ b/docs/cn/best_practice.md @@ -45,8 +45,8 @@ Producer的send方法本身支持内部重试，重试逻辑如下： -- 至多重试2次（同步发送为2次，异步发送为0次）。 -- 如果发送失败，则轮转到下一个Broker。这个方法的总耗时时间不超过sendMsgTimeout设置的值，默认10s。 +- 至多重试2次。 +- 如果同步模式发送失败，则轮转到下一个Broker，如果异步模式发送失败，则只会在当前Broker进行重试。这个方法的总耗时时间不超过sendMsgTimeout设置的值，默认10s。 - 如果本身向broker发送消息产生超时异常，就不会再重试。 以上策略也是在一定程度上保证了消息可以发送成功。如果业务对消息可靠性要求比较高，建议应用增加相应的重试逻辑：比如调用send同步方法发送失败时，则尝试将消息存储到db，然后由后台线程定时重试，确保消息一定到达Broker。 diff --git a/docs/cn/concept.md b/docs/cn/concept.md index 8ad0be6da808a843b8c5fcdf3a03b00c1bc8032f..75532e89da19905d99aaa10fac115269c9c7c196 100644 --- a/docs/cn/concept.md +++ b/docs/cn/concept.md @@ -38,7 +38,7 @@ RocketMQ主要由 Producer、Broker、Consumer 三部分组成，其中Producer 广播消费模式下，相同Consumer Group的每个Consumer实例都接收全量的消息。 ## 13 普通顺序消息（Normal Ordered Message） -普通顺序消费模式下，消费者通过同一个消费队列收到的消息是有顺序的，不同消息队列收到的消息则可能是无顺序的。 +普通顺序消费模式下，消费者通过同一个消息队列（ Topic 分区，称作 Message Queue） 收到的消息是有顺序的，不同消息队列收到的消息则可能是无顺序的。 ## 14 严格顺序消息（Strictly Ordered Message） 严格顺序消息模式下，消费者收到的所有消息均是有顺序的。 diff --git a/docs/cn/design.md b/docs/cn/design.md index dcd39026f2648e85edd3ad2ae54e7d3fa6b76f72..169bc4aaeff3525944b002135f74b491c4f2ea70 100644 --- a/docs/cn/design.md +++ b/docs/cn/design.md @@ -10,7 +10,7 @@ #### 1.1 消息存储整体架构 消息存储架构图中主要有下面三个跟消息存储相关的文件构成。 -(1) CommitLog：消息主体以及元数据的存储主体，存储Producer端写入的消息主体内容,消息内容不是定长的。单个文件大小默认1G ，文件名长度为20位，左边补零，剩余为起始偏移量，比如00000000000000000000代表了第一个文件，起始偏移量为0，文件大小为1G=1073741824；当第一个文件写满了，第二个文件为00000000001073741824，起始偏移量为1073741824，以此类推。消息主要是顺序写入日志文件，当文件满了，写入下一个文件； +(1) CommitLog：消息主体以及元数据的存储主体，存储Producer端写入的消息主体内容,消息内容不是定长的。单个文件大小默认1G, 文件名长度为20位，左边补零，剩余为起始偏移量，比如00000000000000000000代表了第一个文件，起始偏移量为0，文件大小为1G=1073741824；当第一个文件写满了，第二个文件为00000000001073741824，起始偏移量为1073741824，以此类推。消息主要是顺序写入日志文件，当文件满了，写入下一个文件； (2) ConsumeQueue：消息消费队列，引入的目的主要是提高消息消费的性能，由于RocketMQ是基于主题topic的订阅模式，消息消费是针对主题进行的，如果要遍历commitlog文件中根据topic检索消息是非常低效的。Consumer即可根据ConsumeQueue来查找待消费的消息。其中，ConsumeQueue（逻辑消费队列）作为消费消息的索引，保存了指定Topic下的队列消息在CommitLog中的起始物理偏移量offset，消息大小size和消息Tag的HashCode值。consumequeue文件可以看成是基于topic的commitlog索引文件，故consumequeue文件夹的组织方式如下：topic/queue/file三层组织结构，具体存储路径为：$HOME/store/consumequeue/{topic}/{queueId}/{fileName}。同样consumequeue文件采取定长设计，每一个条目共20个字节，分别为8字节的commitlog物理偏移量、4字节的消息长度、8字节tag hashcode，单个文件由30W个条目组成，可以像数组一样随机访问每一个条目，每个ConsumeQueue文件大小约5.72M； @@ -58,7 +58,7 @@ code |int | 请求操作码，应答方根据不同的请求码进行不同的 language | LanguageCode | 请求方实现的语言 | 应答方实现的语言 version | int | 请求方程序的版本 | 应答方程序的版本 opaque | int |相当于requestId，在同一个连接上的不同请求标识码，与响应消息中的相对应 | 应答不做修改直接返回 -flag | int | 区分是普通RPC还是onewayRPC得标志 | 区分是普通RPC还是onewayRPC得标志 +flag | int | 区分是普通RPC还是onewayRPC的标志 | 区分是普通RPC还是onewayRPC的标志 remark | String | 传输自定义文本信息 | 传输自定义文本信息 extFields | HashMap | 请求自定义扩展信息 | 响应自定义扩展信息 @@ -95,7 +95,7 @@ M1 | NettyServerCodecThread_%d | Worker线程池 M2 | RemotingExecutorThread_%d | 业务processor处理线程池 ### 3 消息过滤 -RocketMQ分布式消息队列的消息过滤方式有别于其它MQ中间件，是在Consumer端订阅消息时再做消息过滤的。RocketMQ这么做是在于其Producer端写入消息和Consumer端订阅消息采用分离存储的机制来实现的，Consumer端订阅消息是需要通过ConsumeQueue这个消息消费的逻辑队列拿到一个索引，然后再从CommitLog里面读取真正的消息实体内容，所以说到底也是还绕不开其存储结构。其ConsumeQueue的存储结构如下，可以看到其中有8个字节存储的Message Tag的哈希值，基于Tag的消息过滤正式基于这个字段值的。 +RocketMQ分布式消息队列的消息过滤方式有别于其它MQ中间件，是在Consumer端订阅消息时再做消息过滤的。RocketMQ这么做是在于其Producer端写入消息和Consumer端订阅消息采用分离存储的机制来实现的，Consumer端订阅消息是需要通过ConsumeQueue这个消息消费的逻辑队列拿到一个索引，然后再从CommitLog里面读取真正的消息实体内容，所以说到底也是还绕不开其存储结构。其ConsumeQueue的存储结构如下，可以看到其中有8个字节存储的Message Tag的哈希值，基于Tag的消息过滤正是基于这个字段值的。 ![](image/rocketmq_design_7.png) diff --git a/docs/cn/image/rocketmq_architecture_1.png b/docs/cn/image/rocketmq_architecture_1.png index b8120b27221a42a58f5681b27c4ffea52de1e2aa..1d3e82b354b563e17aa90d64c9f94dd9bfe476b6 100644 Binary files a/docs/cn/image/rocketmq_architecture_1.png and b/docs/cn/image/rocketmq_architecture_1.png differ diff --git a/docs/cn/image/rocketmq_architecture_3.png b/docs/cn/image/rocketmq_architecture_3.png index aa74ed7ac15120112d45b2a2b3213ca452ff79d7..6460f8e1c30ed6e98e245aa29cf79db89f4891cc 100644 Binary files a/docs/cn/image/rocketmq_architecture_3.png and b/docs/cn/image/rocketmq_architecture_3.png differ diff --git a/docs/cn/msg_trace/user_guide.md b/docs/cn/msg_trace/user_guide.md index 828d8c0ce7fc0a7ed9de6db535ae483a96e6ec8a..b6447186ce31f8f367243a0278fffc5ef89b59a9 100644 --- a/docs/cn/msg_trace/user_guide.md +++ b/docs/cn/msg_trace/user_guide.md @@ -50,7 +50,7 @@ RocketMQ的消息轨迹特性支持两种存储轨迹数据的方式： 如果用户不准备将消息轨迹的数据存储于系统级的默认TraceTopic，也可以自己定义并创建用户级的Topic来保存轨迹（即为创建普通的Topic用于保存消息轨迹数据）。下面一节会介绍Client客户端的接口如何支持用户自定义的TraceTopic。 ## 4. 支持消息轨迹的Client客户端实践 -为了尽可能地减少用户业务系统使用RocketMQ消息轨迹特性的改造工作量，作者在设计时候采用对原来接口增加一个开关参数(**enableMsgTrace**)来实现消息轨迹是否开启；并新增一个自定义参(**customizedTraceTopic**)数来实现用户存储消息轨迹数据至自己创建的用户级Topic。 +为了尽可能地减少用户业务系统使用RocketMQ消息轨迹特性的改造工作量，作者在设计时候采用对原来接口增加一个开关参数(**enableMsgTrace**)来实现消息轨迹是否开启；并新增一个自定义参数(**customizedTraceTopic**)来实现用户存储消息轨迹数据至自己创建的用户级Topic。 ### 4.1 发送消息时开启消息轨迹 ``` diff --git a/docs/cn/operation.md b/docs/cn/operation.md index d8c0bb151d4dee4936f52c8a64e74368601e64df..bc2d38527fce649921818ce19928cbe07c01462b 100644 --- a/docs/cn/operation.md +++ b/docs/cn/operation.md @@ -26,7 +26,7 @@ The Name Server boot success... $ nohup sh bin/mqbroker -n localhost:9876 & ### 验证Name Server 是否启动成功，例如Broker的IP为：192.168.1.2，且名称为broker-a -$ tail -f ~/logs/rocketmqlogs/Broker.log +$ tail -f ~/logs/rocketmqlogs/broker.log The broker[broker-a, 192.169.1.2:10911] boot success... ``` diff --git a/docs/cn/rpc_request.md b/docs/cn/rpc_request.md index a8fe802a15293d7c3078bf71eea18d2f642be52f..ffd2bb87e36f221afb40bad4c8becf98f9704b16 100644 --- a/docs/cn/rpc_request.md +++ b/docs/cn/rpc_request.md @@ -4,7 +4,7 @@ ## 1 使用场景 随着服务规模的扩大，单机服务无法满足性能和容量的要求，此时需要将服务拆分为更小粒度的服务或者部署多个服务实例构成集群来提供服务。在分布式场景下，RPC是最常用的联机调用的方式。 -在构建分布式应用时，有些领域，例如金融服务领域，常常使用消息队列来构建服务总线，实现联机调用的目的。消息队列的主要场景是解耦、削峰填谷，在联机调用的场景下，需要将服务的调用抽象成基于消息的交互，并增强同步调用的这种交互逻辑。为了更好地支持消息队列在联机调用场景下的应用，rocketmq-4.7.0推出了“Request-Reply”特性来支持RPC调用。 +在构建分布式应用时，有些领域，例如金融服务领域，常常使用消息队列来构建服务总线，实现联机调用的目的。消息队列的主要场景是解耦、削峰填谷，在联机调用的场景下，需要将服务的调用抽象成基于消息的交互，并增强同步调用的这种交互逻辑。为了更好地支持消息队列在联机调用场景下的应用，rocketmq-4.6.0推出了“Request-Reply”特性来支持RPC调用。 ## 2 设计思路 在rocketmq中，整个同步调用主要包括两个过程： diff --git a/docs/en/Design_LoadBlancing.md b/docs/en/Design_LoadBlancing.md index e6bb66f8ac35710a4708063f8fdb55f0634054cb..971e8b5dd45b73544d3fbc8b389ba010ad052387 100644 --- a/docs/en/Design_LoadBlancing.md +++ b/docs/en/Design_LoadBlancing.md @@ -1,22 +1,24 @@ ## Load Balancing Load balancing in RocketMQ is accomplished on Client side. Specifically, it can be divided into load balancing at Producer side when sending messages and load balancing at Constumer side when subscribing messages. -### 1 Producer Load Balancing +### Producer Load Balancing When the Producer sends a message, it will first find the specified TopicPublishInfo according to Topic. After getting the routing information of TopicPublishInfo, the RocketMQ client will select a queue (MessageQueue) from the messageQueue List in TopicPublishInfo to send the message by default.Specific fault-tolerant strategies are defined in the MQFaultStrategy class. Here is a sendLatencyFaultEnable switch variable, which, if turned on, filters out the Broker agent of not available on the basis of randomly gradually increasing modular arithmetic selection. The so-called "latencyFault Tolerance" refers to a certain period of time to avoid previous failures. For example, if the latency of the last request exceeds 550 Lms, it will evade 3000 Lms; if it exceeds 1000L, it will evade 60000 L; if it is closed, it will choose a queue (MessageQueue) to send messages by randomly gradually increasing modular arithmetic, and the latencyFault Tolerance mechanism is the key to achieve high availability of message sending. -### 2 Consumer Load Balancing +### Consumer Load Balancing In RocketMQ, the two consumption modes (Push/Pull) on the Consumer side are both based on the pull mode to get the message, while in the Push mode it is only a kind of encapsulation of the pull mode, which is essentially implemented as the message pulling thread after pulling a batch of messages from the server. After submitting to the message consuming thread pool, it continues to try again to pull the message to the server. If the message is not pulled, the pull is delayed and continues. In both pull mode based consumption patterns (Push/Pull), the Consumer needs to know which message queue - queue from the Broker side to get the message. Therefore, it is necessary to do load balancing on the Consumer side, that is, which Consumer consumption is allocated to the same ConsumerGroup by more than one MessageQueue on the Broker side. - 1， Heartbeat Packet Sending on Consumer side +#### 1 Heartbeat Packet Sending on Consumer side After Consumer is started, it continuously sends heartbeat packets to all Broker instances in the RocketMQ cluster via timing task (which contains the message consumption group name, subscription relationship collection,Message communication mode and the value of the client id,etc). After receiving the heartbeat message from Consumer, Broker side maintains it in Consumer Manager's local caching variable—consumerTable, At the same time, the encapsulated client network channel information is stored in the local caching variable—channelInfoTable, which can provide metadata information for the later load balancing of Consumer. -2，Core Class for Load Balancing on Consumer side—RebalanceImpl +#### 2 Core Class for Load Balancing on Consumer side—RebalanceImpl Starting the MQClientInstance instance in the startup process of the Consumer instance will complete the start of the load balancing service thread-RebalanceService (executed every 20 s). By looking at the source code, we can find that the run () method of the RebalanceService thread calls the rebalanceByTopic () method of the RebalanceImpl class, which is the core of the Consumer end load balancing. Here, rebalanceByTopic () method will do different logical processing depending on whether the consumer communication type is "broadcast mode" or "cluster mode". Here we mainly look at the main processing flow in cluster mode: -(1) Get the message consumption queue set (mqSet) under the Topic from the local cache variable—topicSubscribeInfoTable of the rebalanceImpl instance. -(2) Call mQClientFactory. findConsumerIdList () method to send a RPC communication request to Broker side to obtain the consumer Id list under the consumer group based on the parameters of topic and consumer group (consumer table constructed by Broker side based on the heartbeat data reported by the front consumer side responds and returns, business request code: GET_CONSUMER_LIST_BY_GROUP); -(3) First, the message consumption queue and the consumer Id under Topic are sorted, then the message queue to be pulled is calculated by using the message queue allocation strategy algorithm (default: the average allocation algorithm of the message queue). The average allocation algorithm here is similar to the paging algorithm. It ranks all MessageQueues like records. It ranks all consumers like pages. It calculates the average size of each page and the range of each page record. Finally, it traverses the whole range and calculates the records that the current consumer should allocate to (MessageQueue here). +##### 1) Get the message consumption queue set (mqSet) under the Topic from the local cache variable—topicSubscribeInfoTable of the rebalanceImpl instance. +##### 2) Call mQClientFactory. findConsumerIdList () method to send a RPC communication request to Broker side to obtain the consumer Id list under the consumer group based on the parameters of topic and consumer group (consumer table constructed by Broker side based on the heartbeat data reported by the front consumer side responds and returns, business request code: GET_CONSUMER_LIST_BY_GROUP); +##### 3) First, the message consumption queue and the consumer Id under Topic are sorted, then the message queue to be pulled is calculated by using the message queue allocation strategy algorithm (default: the average allocation algorithm of the message queue). The average allocation algorithm here is similar to the paging algorithm. It ranks all MessageQueues like records. It ranks all consumers like pages. It calculates the average size of each page and the range of each page record. Finally, it traverses the whole range and calculates the records that the current consumer should allocate to (MessageQueue here). ![Image text](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_8.png) -(4) Then, the updateProcessQueueTableInRebalance () method is invoked, which first compares the allocated message queue set (mqSet) with processQueueTable for filtering. + + +##### 4) Then, the updateProcessQueueTableInRebalance () method is invoked, which first compares the allocated message queue set (mqSet) with processQueueTable for filtering. ![Image text](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_9.png) - The red part of the processQueueTable annotation in the figure above @@ -37,6 +39,6 @@ Starting the MQClientInstance instance in the startup process of the Consumer in removeUnnecessaryMessageQueue () method to try to remove Entry as above; -Finally, a ProcessQueue object is created for each MessageQueue in the filtered message queue set (mqSet) and stored in the processQueueTable queue of RebalanceImpl (where the computePullFromWhere (MessageQueue mq) method of the RebalanceImpl instance is invoked to obtain the next progress consumption value offset of the MessageQueue object, which is then populated into the attribute of pullRequest object to be created next time.), and create pull request object—pullRequest to add to pull list—pullRequestList, and finally execute dispatchPullRequest () method. PullRequest object of Pull message is put into the blocking queue pullRequestQueue of PullMessageService service thread in turn, and the request of Pull message is sent to Broker end after the service thread takes out. Among them, we can focus on the contrast, RebalancePushImpl and RebalancePullImpl two implementation classes dispatchPullRequest () method is different, the method in RebalancePullImpl class is empty, thus answering the last question in the previous article. +Finally, a ProcessQueue object is created for each MessageQueue in the filtered message queue set (mqSet) and stored in the processQueueTable queue of RebalanceImpl (where the computePullFromWhere (MessageQueue mq) method of the RebalanceImpl instance is invoked to obtain the next progress consumption value offset of the MessageQueue object, which is then populated into the attribute of pullRequest object to be created next time.), and create pull request object—pullRequest to add to pull list—pullRequestList, and finally execute dispatchPullRequest () method. PullRequest object of Pull message is put into the blocking queue pullRequestQueue of PullMessageService service thread in turn, and the request of Pull message is sent to Broker end after the service thread takes out. The core design idea of message consumption queue is that a message consumption queue can only be consumed by one consumer in the same consumer group at the same time, and a message consumer can consume multiple message queues at the same time. diff --git a/docs/en/Example_Filter.md b/docs/en/Example_Filter.md index caf34cb7d5009f699e668029c8d3e7c5cfc76e12..8c95a9110618d06786da2194336c74af4a901d3f 100644 --- a/docs/en/Example_Filter.md +++ b/docs/en/Example_Filter.md @@ -74,7 +74,7 @@ Use `MessageSelector.bySql` to select messages through SQL when consuming. ```java DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_4"); // only subsribe messages have property a, also a >=0 and a <= 3 -consumer.subscribe("TopicTest", MessageSelector.bySql("a between 0 and 3"); +consumer.subscribe("TopicTest", MessageSelector.bySql("a between 0 and 3")); consumer.registerMessageListener(new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { diff --git a/docs/en/Example_Transaction.md b/docs/en/Example_Transaction.md index a35b040f821b39a30b065835bd4964141a4d86c9..5d1cff9320e90a352bc62784aba9ed6de38ab68e 100644 --- a/docs/en/Example_Transaction.md +++ b/docs/en/Example_Transaction.md @@ -89,7 +89,7 @@ public class TransactionListenerImpl implements TransactionListener { ## 3 Usage Constraint 1. Messages of the transactional have no schedule and batch support. 2. In order to avoid a single message being checked too many times and lead to half queue message accumulation, we limited the number of checks for a single message to 15 times by default, but users can change this limit by change the ```transactionCheckMax``` parameter in the configuration of the broker, if one message has been checked over ```transactionCheckMax``` times, broker will discard this message and print an error log at the same time by default. Users can change this behavior by override the ```AbstractTransactionalMessageCheckListener``` class. -3. A transactional message will be checked after a certain period of time that determined by parameter g```transactionTimeout``` in the configuration of the broker. And users also can change this limit by set user property ```CHECK_IMMUNITY_TIME_IN_SECONDS``` when sending transactional message, this parameter takes precedence over the ```transactionTimeout``` parameter. +3. A transactional message will be checked after a certain period of time that determined by parameter ```transactionTimeout``` in the configuration of the broker. And users also can change this limit by set user property ```CHECK_IMMUNITY_TIME_IN_SECONDS``` when sending transactional message, this parameter takes precedence over the ```transactionTimeout``` parameter. 4. A transactional message maybe checked or consumed more than once. 5. Committed message reput to the user’s target topic may fail. Currently, it depends on the log record. High availability is ensured by the high availability mechanism of RocketMQ itself. If you want to ensure that the transactional message isn’t lost and the transaction integrity is guaranteed, it is recommended to use synchronous double write. mechanism. 6. Producer IDs of transactional messages cannot be shared with producer IDs of other types of messages. Unlike other types of message, transactional messages allow backward queries. MQ Server query clients by their Producer IDs. diff --git a/docs/en/image/rocketmq_architecture_1.png b/docs/en/image/rocketmq_architecture_1.png index 548c8752a8409bcf7b0f591f293a3bb5bf9435ad..1d3e82b354b563e17aa90d64c9f94dd9bfe476b6 100644 Binary files a/docs/en/image/rocketmq_architecture_1.png and b/docs/en/image/rocketmq_architecture_1.png differ diff --git a/docs/en/image/rocketmq_architecture_3.png b/docs/en/image/rocketmq_architecture_3.png index aa74ed7ac15120112d45b2a2b3213ca452ff79d7..6460f8e1c30ed6e98e245aa29cf79db89f4891cc 100644 Binary files a/docs/en/image/rocketmq_architecture_3.png and b/docs/en/image/rocketmq_architecture_3.png differ diff --git a/example/pom.xml b/example/pom.xml index dba83fe6d26a18728c107708a05a8608e162796c..81f7515dbe332d8bbb6798bc2beb28f4f0e647b6 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -17,9 +17,9 @@ - org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + org.apache.rocketmq + 4.9.1-SNAPSHOT 4.0.0 @@ -36,6 +36,14 @@ ${project.groupId} rocketmq-srvutil + + ${project.groupId} + rocketmq-openmessaging + + + ${project.groupId} + rocketmq-acl + ch.qos.logback logback-classic @@ -45,18 +53,14 @@ javassist - io.openmessaging - openmessaging-api + io.jaegertracing + jaeger-core + 1.6.0 - org.apache.rocketmq - rocketmq-openmessaging - 4.7.1-SNAPSHOT - - - org.apache.rocketmq - rocketmq-acl - 4.7.1-SNAPSHOT + io.jaegertracing + jaeger-client + 1.6.0 diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/AclClient.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/AclClient.java new file mode 100644 index 0000000000000000000000000000000000000000..04ef5d5ef6b7c255cc859f897f04b32fbd49482c --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/AclClient.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.benchmark; + +import org.apache.rocketmq.acl.common.AclClientRPCHook; +import org.apache.rocketmq.acl.common.SessionCredentials; +import org.apache.rocketmq.remoting.RPCHook; + +public class AclClient { + + private static final String ACL_ACCESS_KEY = "rocketmq2"; + + private static final String ACL_SECRET_KEY = "12345678"; + + static RPCHook getAclRPCHook() { + return new AclClientRPCHook(new SessionCredentials(ACL_ACCESS_KEY,ACL_SECRET_KEY)); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java new file mode 100644 index 0000000000000000000000000000000000000000..843b84bf8ed6d0f9de83e770bfdbd6069003fe6b --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java @@ -0,0 +1,411 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.benchmark; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.collections.CollectionUtils; +import org.apache.rocketmq.acl.common.AclClientRPCHook; +import org.apache.rocketmq.acl.common.SessionCredentials; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.srvutil.ServerUtil; + +public class BatchProducer { + + public static void main(String[] args) throws MQClientException, UnsupportedEncodingException { + + Options options = ServerUtil.buildCommandlineOptions(new Options()); + CommandLine commandLine = ServerUtil.parseCmdLine("benchmarkBatchProducer", args, buildCommandlineOptions(options), new PosixParser()); + if (null == commandLine) { + System.exit(-1); + } + + final String namesrv = getOptionValue(commandLine, 'n', "127.0.0.1:9876"); + final String topic = getOptionValue(commandLine, 't', "BenchmarkTest"); + final int threadCount = getOptionValue(commandLine, 'w', 64); + final int messageSize = getOptionValue(commandLine, 's', 128); + final int batchSize = getOptionValue(commandLine, 'b', 16); + final boolean keyEnable = getOptionValue(commandLine, 'k', false); + final int propertySize = getOptionValue(commandLine, 'p', 0); + final int tagCount = getOptionValue(commandLine, 'l', 0); + final boolean msgTraceEnable = getOptionValue(commandLine, 'm', false); + final boolean aclEnable = getOptionValue(commandLine, 'a', false); + final String ak = getOptionValue(commandLine, 'c', "rocketmq2"); + final String sk = getOptionValue(commandLine, 'e', "12346789"); + + System.out.printf("topic: %s threadCount: %d messageSize: %d batchSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s%n", + topic, threadCount, messageSize, batchSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable); + + final StatsBenchmarkBatchProducer statsBenchmark = new StatsBenchmarkBatchProducer(); + statsBenchmark.start(); + + final DefaultMQProducer producer = initInstance(namesrv, msgTraceEnable, aclEnable, ak, sk); + producer.start(); + + final InternalLogger log = ClientLogger.getLog(); + final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount); + for (int i = 0; i < threadCount; i++) { + sendThreadPool.execute(new Runnable() { + @Override + public void run() { + while (true) { + List msgs; + + try { + msgs = buildBathMessage(batchSize, messageSize, topic); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + return; + } + + if (CollectionUtils.isEmpty(msgs)) { + return; + } + + try { + long beginTimestamp = System.currentTimeMillis(); + long sendSucCount = statsBenchmark.getSendMessageSuccessCount().get(); + + setKeys(keyEnable, msgs, String.valueOf(beginTimestamp / 1000)); + setTags(tagCount, msgs, sendSucCount); + setProperties(propertySize, msgs); + SendResult sendResult = producer.send(msgs); + if (sendResult.getSendStatus() == SendStatus.SEND_OK) { + statsBenchmark.getSendRequestSuccessCount().incrementAndGet(); + statsBenchmark.getSendMessageSuccessCount().addAndGet(msgs.size()); + } else { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size()); + } + long currentRT = System.currentTimeMillis() - beginTimestamp; + statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT); + long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); + while (currentRT > prevMaxRT) { + boolean updated = statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT); + if (updated) { + break; + } + + prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); + } + } catch (RemotingException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size()); + log.error("[BENCHMARK_PRODUCER] Send Exception", e); + + try { + Thread.sleep(3000); + } catch (InterruptedException ignored) { + } + } catch (InterruptedException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size()); + try { + Thread.sleep(3000); + } catch (InterruptedException e1) { + } + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size()); + log.error("[BENCHMARK_PRODUCER] Send Exception", e); + } catch (MQClientException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size()); + log.error("[BENCHMARK_PRODUCER] Send Exception", e); + } catch (MQBrokerException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + statsBenchmark.getSendMessageFailedCount().addAndGet(msgs.size()); + log.error("[BENCHMARK_PRODUCER] Send Exception", e); + try { + Thread.sleep(3000); + } catch (InterruptedException ignored) { + } + } + } + } + }); + } + } + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("w", "threadCount", true, "Thread count, Default: 64"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "messageSize", true, "Message Size, Default: 128"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("b", "batchSize", true, "Batch Size, Default: 16"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("k", "keyEnable", true, "Message Key Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("t", "topic", true, "Topic name, Default: BenchmarkTest"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("l", "tagCount", true, "Tag count, Default: 0"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("m", "msgTraceEnable", true, "Message Trace Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("a", "aclEnable", true, "Acl Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "accessKey", true, "Acl Access Key, Default: rocketmq2"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("e", "secretKey", true, "Acl Secret Key, Default: 123456789"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "propertySize", true, "Property Size, Default: 0"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("n", "namesrv", true, "name server, Default: 127.0.0.1:9876"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + private static String getOptionValue(CommandLine commandLine, char key, String defaultValue) { + if (commandLine.hasOption(key)) { + return commandLine.getOptionValue(key).trim(); + } + return defaultValue; + } + + private static int getOptionValue(CommandLine commandLine, char key, int defaultValue) { + if (commandLine.hasOption(key)) { + return Integer.parseInt(commandLine.getOptionValue(key).trim()); + } + return defaultValue; + } + + private static boolean getOptionValue(CommandLine commandLine, char key, boolean defaultValue) { + if (commandLine.hasOption(key)) { + return Boolean.parseBoolean(commandLine.getOptionValue(key).trim()); + } + return defaultValue; + } + + private static List buildBathMessage(int batchSize, int messageSize, + String topic) throws UnsupportedEncodingException { + List batchMessage = new ArrayList<>(batchSize); + + for (int i = 0; i < batchSize; i++) { + Message msg = new Message(); + msg.setTopic(topic); + + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < messageSize; j += 10) { + sb.append("hello baby"); + } + + msg.setBody(sb.toString().getBytes(RemotingHelper.DEFAULT_CHARSET)); + batchMessage.add(msg); + } + + return batchMessage; + } + + private static void setKeys(boolean keyEnable, List msgs, String keys) { + if (!keyEnable) { + return; + } + + for (Message msg : msgs) { + msg.setKeys(keys); + } + } + + private static void setTags(int tagCount, List msgs, long startTagId) { + if (tagCount <= 0) { + return; + } + + long tagId = startTagId % tagCount; + for (Message msg : msgs) { + msg.setTags(String.format("tag%d", tagId++)); + } + } + + private static void setProperties(int propertySize, List msgs) { + if (propertySize <= 0) { + return; + } + + for (Message msg : msgs) { + if (msg.getProperties() != null) { + msg.getProperties().clear(); + } + + int startValue = (new Random(System.currentTimeMillis())).nextInt(100); + int size = 0; + for (int i = 0; ; i++) { + String prop1 = "prop" + i, prop1V = "hello" + startValue; + msg.putUserProperty(prop1, prop1V); + size += prop1.length() + prop1V.length(); + if (size > propertySize) { + break; + } + startValue++; + } + } + } + + private static DefaultMQProducer initInstance(String namesrv, boolean traceEnable, boolean aclEnable, String ak, + String sk) { + RPCHook rpcHook = aclEnable ? new AclClientRPCHook(new SessionCredentials(ak, sk)) : null; + final DefaultMQProducer producer = new DefaultMQProducer("benchmark_batch_producer", rpcHook, traceEnable, null); + producer.setInstanceName(Long.toString(System.currentTimeMillis())); + + producer.setNamesrvAddr(namesrv); + producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE); + return producer; + } +} + +class StatsBenchmarkBatchProducer { + + private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L); + + private final AtomicLong sendRequestFailedCount = new AtomicLong(0L); + + private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L); + + private final AtomicLong sendMessageMaxRT = new AtomicLong(0L); + + private final AtomicLong sendMessageSuccessCount = new AtomicLong(0L); + + private final AtomicLong sendMessageFailedCount = new AtomicLong(0L); + + private final Timer timer = new Timer("BenchmarkTimerThread", true); + + private final LinkedList snapshotList = new LinkedList<>(); + + public Long[] createSnapshot() { + Long[] snap = new Long[] { + System.currentTimeMillis(), + this.sendRequestSuccessCount.get(), + this.sendRequestFailedCount.get(), + this.sendMessageSuccessCount.get(), + this.sendMessageFailedCount.get(), + this.sendMessageSuccessTimeTotal.get(), + }; + + return snap; + } + + public AtomicLong getSendRequestSuccessCount() { + return sendRequestSuccessCount; + } + + public AtomicLong getSendRequestFailedCount() { + return sendRequestFailedCount; + } + + public AtomicLong getSendMessageSuccessTimeTotal() { + return sendMessageSuccessTimeTotal; + } + + public AtomicLong getSendMessageMaxRT() { + return sendMessageMaxRT; + } + + public AtomicLong getSendMessageSuccessCount() { + return sendMessageSuccessCount; + } + + public AtomicLong getSendMessageFailedCount() { + return sendMessageFailedCount; + } + + public void start() { + + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + snapshotList.addLast(createSnapshot()); + if (snapshotList.size() > 10) { + snapshotList.removeFirst(); + } + } + }, 1000, 1000); + + timer.scheduleAtFixedRate(new TimerTask() { + private void printStats() { + if (snapshotList.size() >= 10) { + Long[] begin = snapshotList.getFirst(); + Long[] end = snapshotList.getLast(); + + final long sendTps = (long) (((end[1] - begin[1]) / (double) (end[0] - begin[0])) * 1000L); + final long sendMps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); + final double averageRT = (end[5] - begin[5]) / (double) (end[1] - begin[1]); + final double averageMsgRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); + + System.out.printf("Current Time: %s Send TPS: %d Send MPS: %d Max RT(ms): %d Average RT(ms): %7.3f Average Message RT(ms): %7.3f Send Failed: %d Send Message Failed: %d%n", + System.currentTimeMillis(), sendTps, sendMps, getSendMessageMaxRT().get(), averageRT, averageMsgRT, end[2], end[4]); + } + } + + @Override + public void run() { + try { + this.printStats(); + } catch (Exception e) { + e.printStackTrace(); + } + } + }, 10000, 10000); + } + + public void shutdown() { + timer.cancel(); + } +} \ No newline at end of file diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java index 08897faa2636359e8bb5c982afd71c7a265fbb4a..8a6429b2aa20ea00d5e7e51eb8824163e042d141 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java @@ -34,10 +34,12 @@ import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.srvutil.ServerUtil; public class Consumer { @@ -50,17 +52,22 @@ public class Consumer { } final String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : "BenchmarkTest"; + final int threadCount = commandLine.hasOption('w') ? Integer.parseInt(commandLine.getOptionValue('w')) : 20; final String groupPrefix = commandLine.hasOption('g') ? commandLine.getOptionValue('g').trim() : "benchmark_consumer"; - final String isSuffixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : "true"; + final String isSuffixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : "false"; final String filterType = commandLine.hasOption('f') ? commandLine.getOptionValue('f').trim() : null; final String expression = commandLine.hasOption('e') ? commandLine.getOptionValue('e').trim() : null; final double failRate = commandLine.hasOption('r') ? Double.parseDouble(commandLine.getOptionValue('r').trim()) : 0.0; + final boolean msgTraceEnable = commandLine.hasOption('m') && Boolean.parseBoolean(commandLine.getOptionValue('m')); + final boolean aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a')); + String group = groupPrefix; if (Boolean.parseBoolean(isSuffixEnable)) { group = groupPrefix + "_" + (System.currentTimeMillis() % 100); } - System.out.printf("topic: %s, group: %s, suffix: %s, filterType: %s, expression: %s%n", topic, group, isSuffixEnable, filterType, expression); + System.out.printf("topic: %s, threadCount %d, group: %s, suffix: %s, filterType: %s, expression: %s, msgTraceEnable: %s, aclEnable: %s%n", + topic, threadCount, group, isSuffixEnable, filterType, expression, msgTraceEnable, aclEnable); final StatsBenchmarkConsumer statsBenchmarkConsumer = new StatsBenchmarkConsumer(); @@ -95,8 +102,8 @@ public class Consumer { statsBenchmarkConsumer.getBorn2ConsumerMaxRT().set(0); statsBenchmarkConsumer.getStore2ConsumerMaxRT().set(0); - System.out.printf("TPS: %d FAIL: %d AVG(B2C) RT: %7.3f AVG(S2C) RT: %7.3f MAX(B2C) RT: %d MAX(S2C) RT: %d%n", - consumeTps, failCount, averageB2CRT, averageS2CRT, b2cMax, s2cMax + System.out.printf("Current Time: %s TPS: %d FAIL: %d AVG(B2C) RT(ms): %7.3f AVG(S2C) RT(ms): %7.3f MAX(B2C) RT(ms): %d MAX(S2C) RT(ms): %d%n", + System.currentTimeMillis(), consumeTps, failCount, averageB2CRT, averageS2CRT, b2cMax, s2cMax ); } } @@ -111,11 +118,14 @@ public class Consumer { } }, 10000, 10000); - DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group); + RPCHook rpcHook = aclEnable ? AclClient.getAclRPCHook() : null; + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group, rpcHook, new AllocateMessageQueueAveragely(), msgTraceEnable, null); if (commandLine.hasOption('n')) { String ns = commandLine.getOptionValue('n'); consumer.setNamesrvAddr(ns); } + consumer.setConsumeThreadMin(threadCount); + consumer.setConsumeThreadMax(threadCount); consumer.setInstanceName(Long.toString(System.currentTimeMillis())); if (filterType == null || expression == null) { @@ -172,11 +182,15 @@ public class Consumer { opt.setRequired(false); options.addOption(opt); + opt = new Option("w", "threadCount", true, "Thread count, Default: 20"); + opt.setRequired(false); + options.addOption(opt); + opt = new Option("g", "group", true, "Consumer group name, Default: benchmark_consumer"); opt.setRequired(false); options.addOption(opt); - opt = new Option("p", "group prefix enable", true, "Consumer group name, Default: false"); + opt = new Option("p", "group suffix enable", true, "Consumer group suffix enable, Default: false"); opt.setRequired(false); options.addOption(opt); @@ -192,6 +206,14 @@ public class Consumer { opt.setRequired(false); options.addOption(opt); + opt = new Option("m", "msgTraceEnable", true, "Message Trace Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("a", "aclEnable", true, "Acl Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + return options; } diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java index ce2b83f97755c51a3fabda1f22ef13d276245929..32d4b9f7462eb9e3a5ad373295fffc6932970e3a 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java @@ -17,12 +17,14 @@ package org.apache.rocketmq.example.benchmark; import java.io.UnsupportedEncodingException; +import java.util.Arrays; import java.util.LinkedList; import java.util.Random; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.cli.CommandLine; @@ -35,11 +37,13 @@ import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.srvutil.ServerUtil; public class Producer { + public static void main(String[] args) throws MQClientException, UnsupportedEncodingException { Options options = ServerUtil.buildCommandlineOptions(new Options()); @@ -53,8 +57,15 @@ public class Producer { final int messageSize = commandLine.hasOption('s') ? Integer.parseInt(commandLine.getOptionValue('s')) : 128; final boolean keyEnable = commandLine.hasOption('k') && Boolean.parseBoolean(commandLine.getOptionValue('k')); final int propertySize = commandLine.hasOption('p') ? Integer.parseInt(commandLine.getOptionValue('p')) : 0; + final int tagCount = commandLine.hasOption('l') ? Integer.parseInt(commandLine.getOptionValue('l')) : 0; + final boolean msgTraceEnable = commandLine.hasOption('m') && Boolean.parseBoolean(commandLine.getOptionValue('m')); + final boolean aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a')); + final long messageNum = commandLine.hasOption('q') ? Long.parseLong(commandLine.getOptionValue('q')) : 0; + final boolean delayEnable = commandLine.hasOption('d') && Boolean.parseBoolean(commandLine.getOptionValue('d')); + final int delayLevel = commandLine.hasOption('e') ? Integer.parseInt(commandLine.getOptionValue('e')) : 1; - System.out.printf("topic %s threadCount %d messageSize %d keyEnable %s%n", topic, threadCount, messageSize, keyEnable); + System.out.printf("topic: %s threadCount: %d messageSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s messageQuantity: %d%n delayEnable: %s%n delayLevel: %s%n", + topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable, messageNum, delayEnable, delayLevel); final InternalLogger log = ClientLogger.getLog(); @@ -66,6 +77,16 @@ public class Producer { final LinkedList snapshotList = new LinkedList(); + final long[] msgNums = new long[threadCount]; + + if (messageNum > 0) { + Arrays.fill(msgNums, messageNum / threadCount); + long mod = messageNum % threadCount; + if (mod > 0) { + msgNums[0] += mod; + } + } + timer.scheduleAtFixedRate(new TimerTask() { @Override public void run() { @@ -79,14 +100,7 @@ public class Producer { timer.scheduleAtFixedRate(new TimerTask() { private void printStats() { if (snapshotList.size() >= 10) { - Long[] begin = snapshotList.getFirst(); - Long[] end = snapshotList.getLast(); - - final long sendTps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); - final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); - - System.out.printf("Send TPS: %d Max RT: %d Average RT: %7.3f Send Failed: %d Response Failed: %d%n", - sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]); + doPrintStats(snapshotList, statsBenchmark, false); } } @@ -100,7 +114,8 @@ public class Producer { } }, 10000, 10000); - final DefaultMQProducer producer = new DefaultMQProducer("benchmark_producer"); + RPCHook rpcHook = aclEnable ? AclClient.getAclRPCHook() : null; + final DefaultMQProducer producer = new DefaultMQProducer("benchmark_producer", rpcHook, msgTraceEnable, null); producer.setInstanceName(Long.toString(System.currentTimeMillis())); if (commandLine.hasOption('n')) { @@ -113,9 +128,14 @@ public class Producer { producer.start(); for (int i = 0; i < threadCount; i++) { + final long msgNumLimit = msgNums[i]; + if (messageNum > 0 && msgNumLimit == 0) { + break; + } sendThreadPool.execute(new Runnable() { @Override public void run() { + int num = 0; while (true) { try { final Message msg; @@ -129,6 +149,13 @@ public class Producer { if (keyEnable) { msg.setKeys(String.valueOf(beginTimestamp / 1000)); } + if (delayEnable) { + msg.setDelayTimeLevel(delayLevel); + } + if (tagCount > 0) { + long sendSucCount = statsBenchmark.getReceiveResponseSuccessCount().get(); + msg.setTags(String.format("tag%d", sendSucCount % tagCount)); + } if (propertySize > 0) { if (msg.getProperties() != null) { msg.getProperties().clear(); @@ -187,10 +214,28 @@ public class Producer { } catch (InterruptedException ignored) { } } + if (messageNum > 0 && ++num >= msgNumLimit) { + break; + } } } }); } + try { + sendThreadPool.shutdown(); + sendThreadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + timer.cancel(); + if (snapshotList.size() > 1) { + doPrintStats(snapshotList, statsBenchmark, true); + } else { + System.out.printf("[Complete] Send Total: %d Send Failed: %d Response Failed: %d%n", + statsBenchmark.getSendRequestSuccessCount().get() + statsBenchmark.getSendRequestFailedCount().get(), + statsBenchmark.getSendRequestFailedCount().get(), statsBenchmark.getReceiveResponseFailedCount().get()); + } + producer.shutdown(); + } catch (InterruptedException e) { + log.error("[Exit] Thread Interrupted Exception", e); + } } public static Options buildCommandlineOptions(final Options options) { @@ -210,6 +255,30 @@ public class Producer { opt.setRequired(false); options.addOption(opt); + opt = new Option("l", "tagCount", true, "Tag count, Default: 0"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("m", "msgTraceEnable", true, "Message Trace Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("a", "aclEnable", true, "Acl Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("q", "messageQuantity", true, "Send message quantity, Default: 0, running forever"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("d", "delayEnable", true, "Delay message Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("e", "delayLevel", true, "Delay message level, Default: 1"); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -226,6 +295,23 @@ public class Producer { return msg; } + + private static void doPrintStats(final LinkedList snapshotList, final StatsBenchmarkProducer statsBenchmark, boolean done) { + Long[] begin = snapshotList.getFirst(); + Long[] end = snapshotList.getLast(); + + final long sendTps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); + final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); + + if (done) { + System.out.printf("[Complete] Send Total: %d Send TPS: %d Max RT(ms): %d Average RT(ms): %7.3f Send Failed: %d Response Failed: %d%n", + statsBenchmark.getSendRequestSuccessCount().get() + statsBenchmark.getSendRequestFailedCount().get(), + sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]); + } else { + System.out.printf("Current Time: %s Send TPS: %d Max RT(ms): %d Average RT(ms): %7.3f Send Failed: %d Response Failed: %d%n", + System.currentTimeMillis(), sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]); + } + } } class StatsBenchmarkProducer { diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java index 3531eb528fdf55de4d82d43fca9b36345903e15a..85af04eabb6c9a9a91b50b6b8988e9b38ecad096 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java @@ -68,6 +68,7 @@ public class TransactionProducer { config.checkUnknownRate = commandLine.hasOption("cu") ? Double.parseDouble(commandLine.getOptionValue("cu")) : 0.0; config.batchId = commandLine.hasOption("b") ? Long.parseLong(commandLine.getOptionValue("b")) : System.currentTimeMillis(); config.sendInterval = commandLine.hasOption("i") ? Integer.parseInt(commandLine.getOptionValue("i")) : 0; + config.aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a')); final ExecutorService sendThreadPool = Executors.newFixedThreadPool(config.threadCount); @@ -104,8 +105,8 @@ public class TransactionProducer { final long dupCheck = end.duplicatedCheck - begin.duplicatedCheck; System.out.printf( - "Send TPS:%5d Max RT:%5d AVG RT:%3.1f Send Failed: %d check: %d unexpectedCheck: %d duplicatedCheck: %d %n", - sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, failCount, checkCount, + "Current Time: %s Send TPS:%5d Max RT(ms):%5d AVG RT(ms):%3.1f Send Failed: %d check: %d unexpectedCheck: %d duplicatedCheck: %d %n", + System.currentTimeMillis(), sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, failCount, checkCount, unexpectedCheck, dupCheck); statsBenchmark.getSendMessageMaxRT().set(0); } @@ -122,7 +123,8 @@ public class TransactionProducer { }, 10000, 10000); final TransactionListener transactionCheckListener = new TransactionListenerImpl(statsBenchmark, config); - final TransactionMQProducer producer = new TransactionMQProducer("benchmark_transaction_producer"); + final TransactionMQProducer producer = + new TransactionMQProducer("benchmark_transaction_producer", config.aclEnable ? AclClient.getAclRPCHook() : null); producer.setInstanceName(Long.toString(System.currentTimeMillis())); producer.setTransactionListener(transactionCheckListener); producer.setDefaultTopicQueueNums(1000); @@ -250,6 +252,10 @@ public class TransactionProducer { opt.setRequired(false); options.addOption(opt); + opt = new Option("a", "aclEnable", true, "Acl Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + return options; } } @@ -432,6 +438,7 @@ class TxSendConfig { double checkUnknownRate; long batchId; int sendInterval; + boolean aclEnable; } class LRUMap extends LinkedHashMap { diff --git a/example/src/main/java/org/apache/rocketmq/example/ordermessage/Producer.java b/example/src/main/java/org/apache/rocketmq/example/ordermessage/Producer.java index 6a6bdc7d710e750f5738ff9a705e3680967e4e24..c5d864fa3f6acae04489cc1a089b9d332c4b9d56 100644 --- a/example/src/main/java/org/apache/rocketmq/example/ordermessage/Producer.java +++ b/example/src/main/java/org/apache/rocketmq/example/ordermessage/Producer.java @@ -16,12 +16,9 @@ */ package org.apache.rocketmq.example.ordermessage; -import java.io.UnsupportedEncodingException; -import java.util.List; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.client.producer.MQProducer; import org.apache.rocketmq.client.producer.MessageQueueSelector; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.message.Message; @@ -29,10 +26,13 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; +import java.io.UnsupportedEncodingException; +import java.util.List; + public class Producer { public static void main(String[] args) throws UnsupportedEncodingException { try { - MQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); + DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); producer.start(); String[] tags = new String[] {"TagA", "TagB", "TagC", "TagD", "TagE"}; diff --git a/example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingProducer.java b/example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingProducer.java new file mode 100644 index 0000000000000000000000000000000000000000..cd9ae2792b667ed6122605775eaaf34055664ca4 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingProducer.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.tracemessage; + +import io.jaegertracing.Configuration; +import io.jaegertracing.internal.samplers.ConstSampler; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.trace.hook.SendMessageOpenTracingHookImpl; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.remoting.common.RemotingHelper; + +public class OpenTracingProducer { + public static void main(String[] args) throws MQClientException { + + Tracer tracer = initTracer(); + + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName"); + producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageOpenTracingHookImpl(tracer)); + producer.start(); + + try { + Message msg = new Message("TopicTest", + "TagA", + "OrderID188", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.send(msg); + System.out.printf("%s%n", sendResult); + + } catch (Exception e) { + e.printStackTrace(); + } + + producer.shutdown(); + } + + private static Tracer initTracer() { + Configuration.SamplerConfiguration samplerConfig = Configuration.SamplerConfiguration.fromEnv() + .withType(ConstSampler.TYPE) + .withParam(1); + Configuration.ReporterConfiguration reporterConfig = Configuration.ReporterConfiguration.fromEnv() + .withLogSpans(true); + + Configuration config = new Configuration("rocketmq") + .withSampler(samplerConfig) + .withReporter(reporterConfig); + GlobalTracer.registerIfAbsent(config.getTracer()); + return config.getTracer(); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingPushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingPushConsumer.java new file mode 100644 index 0000000000000000000000000000000000000000..1d5d8a273b2dbc4d7b29a132356b17d4389c734b --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingPushConsumer.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.tracemessage; + +import io.jaegertracing.Configuration; +import io.jaegertracing.internal.samplers.ConstSampler; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.trace.hook.ConsumeMessageOpenTracingHookImpl; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageExt; + +import java.util.List; + +public class OpenTracingPushConsumer { + public static void main(String[] args) throws InterruptedException, MQClientException { + Tracer tracer = initTracer(); + + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1"); + consumer.getDefaultMQPushConsumerImpl().registerConsumeMessageHook(new ConsumeMessageOpenTracingHookImpl(tracer)); + + consumer.subscribe("TopicTest", "*"); + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + + consumer.setConsumeTimestamp("20181109221800"); + consumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + consumer.start(); + System.out.printf("Consumer Started.%n"); + } + + private static Tracer initTracer() { + Configuration.SamplerConfiguration samplerConfig = Configuration.SamplerConfiguration.fromEnv() + .withType(ConstSampler.TYPE) + .withParam(1); + Configuration.ReporterConfiguration reporterConfig = Configuration.ReporterConfiguration.fromEnv() + .withLogSpans(true); + + Configuration config = new Configuration("rocketmq") + .withSampler(samplerConfig) + .withReporter(reporterConfig); + GlobalTracer.registerIfAbsent(config.getTracer()); + return config.getTracer(); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingTransactionProducer.java b/example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingTransactionProducer.java new file mode 100644 index 0000000000000000000000000000000000000000..514f3ceb7fe6fe0f3491de05f64485072118df37 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/tracemessage/OpenTracingTransactionProducer.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.tracemessage; + +import io.jaegertracing.Configuration; +import io.jaegertracing.internal.samplers.ConstSampler; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.LocalTransactionState; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.TransactionListener; +import org.apache.rocketmq.client.producer.TransactionMQProducer; +import org.apache.rocketmq.client.trace.hook.EndTransactionOpenTracingHookImpl; +import org.apache.rocketmq.client.trace.hook.SendMessageOpenTracingHookImpl; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.common.RemotingHelper; + +import java.io.UnsupportedEncodingException; + +public class OpenTracingTransactionProducer { + public static void main(String[] args) throws MQClientException, InterruptedException { + Tracer tracer = initTracer(); + + TransactionMQProducer producer = new TransactionMQProducer("please_rename_unique_group_name"); + producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageOpenTracingHookImpl(tracer)); + producer.getDefaultMQProducerImpl().registerEndTransactionHook(new EndTransactionOpenTracingHookImpl(tracer)); + + producer.setTransactionListener(new TransactionListener() { + @Override + public LocalTransactionState executeLocalTransaction(Message msg, Object arg) { + return LocalTransactionState.COMMIT_MESSAGE; + } + + @Override + public LocalTransactionState checkLocalTransaction(MessageExt msg) { + return LocalTransactionState.COMMIT_MESSAGE; + } + }); + producer.start(); + + try { + Message msg = new Message("TopicTest", "Tag", "KEY", + "Hello RocketMQ".getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.sendMessageInTransaction(msg, null); + System.out.printf("%s%n", sendResult); + } catch (MQClientException | UnsupportedEncodingException e) { + e.printStackTrace(); + } + + for (int i = 0; i < 100000; i++) { + Thread.sleep(1000); + } + producer.shutdown(); + } + + private static Tracer initTracer() { + Configuration.SamplerConfiguration samplerConfig = Configuration.SamplerConfiguration.fromEnv() + .withType(ConstSampler.TYPE) + .withParam(1); + Configuration.ReporterConfiguration reporterConfig = Configuration.ReporterConfiguration.fromEnv() + .withLogSpans(true); + + Configuration config = new Configuration("rocketmq") + .withSampler(samplerConfig) + .withReporter(reporterConfig); + GlobalTracer.registerIfAbsent(config.getTracer()); + return config.getTracer(); + } +} diff --git a/filter/pom.xml b/filter/pom.xml index 46debc3e38e81e5edcc36b0b42c2cbac7c54d0c0..41bda9d0d22d2c50c10ccdf4b36263e8689d3083 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java index 53f7e1c297e3acea100512b068c1a852e4e09c26..c10250e3bbb896071b3ef08a9247b707930eba88 100644 --- a/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java +++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java @@ -36,8 +36,8 @@ public class SimpleCharStream { * Position in buffer. */ public int bufpos = -1; - protected int bufline[]; - protected int bufcolumn[]; + protected int[] bufline; + protected int[] bufcolumn; protected int column = 0; protected int line = 1; @@ -62,8 +62,8 @@ public class SimpleCharStream { protected void ExpandBuff(boolean wrapAround) { char[] newbuffer = new char[bufsize + 2048]; - int newbufline[] = new int[bufsize + 2048]; - int newbufcolumn[] = new int[bufsize + 2048]; + int[] newbufline = new int[bufsize + 2048]; + int[] newbufcolumn = new int[bufsize + 2048]; try { if (wrapAround) { diff --git a/logappender/pom.xml b/logappender/pom.xml index c4d0b217d343a50aba6ab955114ffefe94374495..cac520b610360308adb90a4a0dbf67ea2ed12d92 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 rocketmq-logappender @@ -42,11 +42,6 @@ log4j-core true - - ch.qos.logback - logback-core - true - ch.qos.logback logback-classic diff --git a/logging/pom.xml b/logging/pom.xml index b4a52a7b0af14e9eb165f8522cb844de67791335..8ebb24304529e907f918e3e77176b200693d6189 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -27,11 +27,6 @@ rocketmq-logging rocketmq-logging ${project.version} - - 1.6 - 1.6 - - org.slf4j diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java index 487682cd44b171e8d3fff8d41d790d1d2383cb50..0dc81d74e6749bcbcdcfc37b2e80e065faf10f2b 100755 --- a/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java @@ -116,7 +116,6 @@ public class Level implements Serializable { if (s.equals(OFF_NAME)) { return Level.OFF; } - if (s.equals(INFO_NAME)) { return Level.INFO; } diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java index f457a69567626f3a8c9b4e852e877976b07d9a13..3ec440bc882bb55b014d567bda8f8447a338955d 100644 --- a/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/LoggingBuilder.java @@ -1201,7 +1201,7 @@ public class LoggingBuilder { public String format(LoggingEvent event) { StringBuilder sb = new StringBuilder(); - SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,sss"); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); String format = simpleDateFormat.format(new Date(event.timeStamp)); sb.append(format); sb.append(" "); diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/SysLogger.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/SysLogger.java index b6d10497782d9714b989e682ea1a2ea2ccc88a7c..aaba4d6e89d01b64abfcbbe6d33551fc82427645 100755 --- a/logging/src/main/java/org/apache/rocketmq/logging/inner/SysLogger.java +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/SysLogger.java @@ -33,13 +33,13 @@ public class SysLogger { public static void debug(String msg) { if (debugEnabled && !quietMode) { - System.out.printf("%s", PREFIX + msg); + System.err.println(PREFIX + msg); } } public static void debug(String msg, Throwable t) { if (debugEnabled && !quietMode) { - System.out.printf("%s", PREFIX + msg); + System.err.println(PREFIX + msg); if (t != null) { t.printStackTrace(System.out); } diff --git a/namesrv/pom.xml b/namesrv/pom.xml index d8ae216588f3fe978da0e95aaeaf2a8a98a44556..01ef5b60fe951dfccff97a80ebd0df8f14cf1b47 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -29,11 +29,11 @@ - org.apache.rocketmq + ${project.groupId} rocketmq-client - org.apache.rocketmq + ${project.groupId} rocketmq-tools @@ -44,10 +44,6 @@ ch.qos.logback logback-classic - - ch.qos.logback - logback-core - org.slf4j slf4j-api diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java index 44a2b6f2610c57ade7000958633eb4c4ddd9d566..f8bc55e7aab38e1f27abc75c674c28eab0a9d02b 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -140,6 +140,11 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen final PutKVConfigRequestHeader requestHeader = (PutKVConfigRequestHeader) request.decodeCommandCustomHeader(PutKVConfigRequestHeader.class); + if (requestHeader.getNamespace() == null || requestHeader.getKey() == null) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("namespace or key is null"); + return response; + } this.namesrvController.getKvConfigManager().putKVConfig( requestHeader.getNamespace(), requestHeader.getKey(), diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java index ecd057a29ac28a9cf736ea7bea51d469afe5e1ff..edef87ce2d7e5d18f00610ad2619a55357d7a5d5 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java @@ -220,7 +220,7 @@ public class RouteInfoManager { queueData.setWriteQueueNums(topicConfig.getWriteQueueNums()); queueData.setReadQueueNums(topicConfig.getReadQueueNums()); queueData.setPerm(topicConfig.getPerm()); - queueData.setTopicSynFlag(topicConfig.getTopicSysFlag()); + queueData.setTopicSysFlag(topicConfig.getTopicSysFlag()); List queueDataList = this.topicQueueTable.get(topicConfig.getTopicName()); if (null == queueDataList) { @@ -684,7 +684,7 @@ public class RouteInfoManager { String topic = topicEntry.getKey(); List queueDatas = topicEntry.getValue(); if (queueDatas != null && queueDatas.size() > 0 - && TopicSysFlag.hasUnitFlag(queueDatas.get(0).getTopicSynFlag())) { + && TopicSysFlag.hasUnitFlag(queueDatas.get(0).getTopicSysFlag())) { topicList.getTopicList().add(topic); } } @@ -710,7 +710,7 @@ public class RouteInfoManager { String topic = topicEntry.getKey(); List queueDatas = topicEntry.getValue(); if (queueDatas != null && queueDatas.size() > 0 - && TopicSysFlag.hasUnitSubFlag(queueDatas.get(0).getTopicSynFlag())) { + && TopicSysFlag.hasUnitSubFlag(queueDatas.get(0).getTopicSysFlag())) { topicList.getTopicList().add(topic); } } @@ -736,8 +736,8 @@ public class RouteInfoManager { String topic = topicEntry.getKey(); List queueDatas = topicEntry.getValue(); if (queueDatas != null && queueDatas.size() > 0 - && !TopicSysFlag.hasUnitFlag(queueDatas.get(0).getTopicSynFlag()) - && TopicSysFlag.hasUnitSubFlag(queueDatas.get(0).getTopicSynFlag())) { + && !TopicSysFlag.hasUnitFlag(queueDatas.get(0).getTopicSysFlag()) + && TopicSysFlag.hasUnitSubFlag(queueDatas.get(0).getTopicSysFlag())) { topicList.getTopicList().add(topic); } } diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 9fcd708ef9b16e18dc64c45f510c31839f51711f..4688c04043633cf98658a7d7d55cd5dad6459d27 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,10 +20,10 @@ rocketmq-all org.apache.rocketmq - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT - 4.0.0 + 4.0.0 rocketmq-openmessaging rocketmq-openmessaging ${project.version} @@ -33,7 +33,7 @@ openmessaging-api - org.apache.rocketmq + ${project.groupId} rocketmq-client diff --git a/pom.xml b/pom.xml index 05f544c7d6ebd9ef7310508536b37e4a3de36c95..b6f1770e2d85c6295c02f5077926e158d1d8f872 100644 --- a/pom.xml +++ b/pom.xml @@ -29,15 +29,11 @@ 2012 org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ - - 3.2.5 - - git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git @@ -106,7 +102,6 @@ ${project.basedir}/../test/target/jacoco-it.exec file:**/generated-sources/**,**/test/** - 2.0.2 @@ -119,7 +114,6 @@ namesrv remoting logappender - example srvutil filter test @@ -127,6 +121,7 @@ openmessaging logging acl + example @@ -182,21 +177,6 @@ true - - maven-javadoc-plugin - 2.10.4 - - UTF-8 - - - - attach-javadocs - - jar - - - - maven-source-plugin 3.0.1 @@ -444,7 +424,7 @@ junit junit - 4.11 + 4.13.2 test @@ -456,19 +436,13 @@ org.mockito mockito-core - 2.23.0 + 3.10.0 test - org.powermock - powermock-module-junit4 - ${powermock.version} - test - - - org.powermock - powermock-api-mockito2 - ${powermock.version} + org.awaitility + awaitility + 4.1.0 test @@ -532,12 +506,17 @@ ${project.groupId} - rocketmq-example + rocketmq-acl ${project.version} ${project.groupId} - rocketmq-acl + rocketmq-openmessaging + ${project.version} + + + ${project.groupId} + rocketmq-example ${project.version} @@ -550,11 +529,6 @@ logback-classic 1.0.13 - - ch.qos.logback - logback-core - 1.0.13 - commons-cli commons-cli @@ -568,7 +542,7 @@ com.alibaba fastjson - 1.2.68 + 1.2.76 org.javassist @@ -623,7 +597,7 @@ commons-validator commons-validator - 1.6 + 1.7 diff --git a/remoting/pom.xml b/remoting/pom.xml index d71f0364a67b89a52e518cfc49bed8f23d02f5af..765e1a93bc0f83a7d59a794619ba356f7cccda9c 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -27,11 +27,6 @@ rocketmq-remoting rocketmq-remoting ${project.version} - - 1.6 - 1.6 - - com.alibaba @@ -45,10 +40,5 @@ ${project.groupId} rocketmq-logging - - io.netty - netty-tcnative-boringssl-static - 1.1.33.Fork26 - diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java index a16940e561c5cec23c9df021c429e1c64d500b88..d936c3bf696ab0399a323ea028e9ee12ee46ce5a 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java @@ -19,6 +19,7 @@ package org.apache.rocketmq.remoting.common; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; +import java.io.File; import java.io.IOException; import java.lang.reflect.Method; import java.net.Inet6Address; @@ -98,6 +99,10 @@ public class RemotingUtil { ArrayList ipv6Result = new ArrayList(); while (enumeration.hasMoreElements()) { final NetworkInterface networkInterface = enumeration.nextElement(); + if (isBridge(networkInterface)) { + continue; + } + final Enumeration en = networkInterface.getInetAddresses(); while (en.hasMoreElements()) { final InetAddress address = en.nextElement(); @@ -160,6 +165,19 @@ public class RemotingUtil { return sb.toString(); } + private static boolean isBridge(NetworkInterface networkInterface) { + try { + if (isLinuxPlatform()) { + String interfaceName = networkInterface.getName(); + File file = new File("/sys/class/net/" + interfaceName + "/bridge"); + return file.exists(); + } + } catch (SecurityException e) { + //Ignore + } + return false; + } + public static SocketChannel connect(SocketAddress remote) { return connect(remote, 1000 * 5); } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index d836c8e0bfba9865215144279ba3ecba040d2b0d..5ba6cfab0a740a46a20d69a104b74094ad81993f 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -84,7 +84,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti private final AtomicReference> namesrvAddrList = new AtomicReference>(); private final AtomicReference namesrvAddrChoosed = new AtomicReference(); private final AtomicInteger namesrvIndex = new AtomicInteger(initValueIndex()); - private final Lock lockNamesrvChannel = new ReentrantLock(); + private final Lock namesrvChannelLock = new ReentrantLock(); private final ExecutorService publicExecutor; @@ -418,7 +418,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti } final List addrList = this.namesrvAddrList.get(); - if (this.lockNamesrvChannel.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + if (this.namesrvChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { try { addr = this.namesrvAddrChoosed.get(); if (addr != null) { @@ -445,7 +445,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti throw new RemotingConnectException(addrList.toString()); } } finally { - this.lockNamesrvChannel.unlock(); + this.namesrvChannelLock.unlock(); } } else { log.warn("getAndCreateNameserverChannel: try to lock name server, but timeout, {}ms", LOCK_TIMEOUT_MILLIS); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java index efbdd52c07387327e5d10bdca8be006c7ef677d1..5003ce39efb03b5a8c75a88d1bd90a33921ee5ff 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java @@ -133,7 +133,7 @@ public class TlsHelper { SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate(); return SslContextBuilder .forServer(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey()) - .sslProvider(SslProvider.JDK) + .sslProvider(provider) .clientAuth(ClientAuth.OPTIONAL) .build(); } else { diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java index 5330c90060da4304d3bba550cdfe2f5415474153..a272d21da41b3c41d58755eccde5f4bf64980fdd 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java @@ -27,8 +27,6 @@ import org.mockito.junit.MockitoJUnitRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.notNull; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -98,7 +96,7 @@ public class NettyRemotingAbstractTest { public void testScanResponseTable() { int dummyId = 1; // mock timeout - ResponseFuture responseFuture = new ResponseFuture(null,dummyId, -1000, new InvokeCallback() { + ResponseFuture responseFuture = new ResponseFuture(null, dummyId, -1000, new InvokeCallback() { @Override public void operationComplete(final ResponseFuture responseFuture) { } diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java index 6b5633df1d74a33c87dbceb3df270b2f1c580224..4b38ce9524c4b6e20a6e365c9541d7f922dfb186 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.remoting.netty; -import java.lang.reflect.Field; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.junit.Test; diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 60090978628a8f65ab2a2ddb72249994f904e7a3..9c4544c0b25e55c1e817aba7c49f2574d0dd0dc6 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index 59a9bfa8f22736ea2832fbdbfb3fba7e58807dfc..23e9763d81402d43340a3db26bf29a654b7e9587 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -31,7 +31,7 @@ io.openmessaging.storage dledger - 0.1 + 0.2.2 org.apache.rocketmq @@ -56,10 +56,5 @@ logback-classic test - - ch.qos.logback - logback-core - test - diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 810110eeb67237342e6005e46fdc29b8e3f6c65f..cce6481b8daa119def971b3c4c2f95d4b3e6502a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -37,6 +37,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.config.BrokerRole; @@ -340,7 +341,7 @@ public class CommitLog { // Timing message processing { String t = propertiesMap.get(MessageConst.PROPERTY_DELAY_TIME_LEVEL); - if (ScheduleMessageService.SCHEDULE_TOPIC.equals(topic) && t != null) { + if (TopicValidator.RMQ_SYS_SCHEDULE_TOPIC.equals(topic) && t != null) { int delayLevel = Integer.parseInt(t); if (delayLevel > this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) { @@ -577,7 +578,7 @@ public class CommitLog { msg.setDelayTimeLevel(this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()); } - topic = ScheduleMessageService.SCHEDULE_TOPIC; + topic = TopicValidator.RMQ_SYS_SCHEDULE_TOPIC; queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()); // Backup real topic, queueId @@ -660,14 +661,18 @@ public class CommitLog { storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).incrementAndGet(); storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes()); - CompletableFuture flushResultFuture = submitFlushRequest(result, putMessageResult, msg); - CompletableFuture replicaResultFuture = submitReplicaRequest(result, putMessageResult, msg); + CompletableFuture flushResultFuture = submitFlushRequest(result, msg); + CompletableFuture replicaResultFuture = submitReplicaRequest(result, msg); return flushResultFuture.thenCombine(replicaResultFuture, (flushStatus, replicaStatus) -> { if (flushStatus != PutMessageStatus.PUT_OK) { - putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT); + putMessageResult.setPutMessageStatus(flushStatus); } if (replicaStatus != PutMessageStatus.PUT_OK) { putMessageResult.setPutMessageStatus(replicaStatus); + if (replicaStatus == PutMessageStatus.FLUSH_SLAVE_TIMEOUT) { + log.error("do sync transfer other node, wait return, but failed, topic: {} tags: {} client address: {}", + msg.getTopic(), msg.getTags(), msg.getBornHostNameString()); + } } return putMessageResult; }); @@ -761,15 +766,18 @@ public class CommitLog { storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).addAndGet(result.getMsgNum()); storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).addAndGet(result.getWroteBytes()); - CompletableFuture flushOKFuture = submitFlushRequest(result, putMessageResult, messageExtBatch); - CompletableFuture replicaOKFuture = submitReplicaRequest(result, putMessageResult, messageExtBatch); + CompletableFuture flushOKFuture = submitFlushRequest(result, messageExtBatch); + CompletableFuture replicaOKFuture = submitReplicaRequest(result, messageExtBatch); return flushOKFuture.thenCombine(replicaOKFuture, (flushStatus, replicaStatus) -> { if (flushStatus != PutMessageStatus.PUT_OK) { - putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT); + putMessageResult.setPutMessageStatus(flushStatus); } - if (replicaStatus != PutMessageStatus.PUT_OK) { putMessageResult.setPutMessageStatus(replicaStatus); + if (replicaStatus == PutMessageStatus.FLUSH_SLAVE_TIMEOUT) { + log.error("do sync transfer other node, wait return, but failed, topic: {} client address: {}", + messageExtBatch.getTopic(), messageExtBatch.getBornHostNameString()); + } } return putMessageResult; }); @@ -799,7 +807,7 @@ public class CommitLog { msg.setDelayTimeLevel(this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()); } - topic = ScheduleMessageService.SCHEDULE_TOPIC; + topic = TopicValidator.RMQ_SYS_SCHEDULE_TOPIC; queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()); // Backup real topic, queueId @@ -899,8 +907,7 @@ public class CommitLog { return putMessageResult; } - public CompletableFuture submitFlushRequest(AppendMessageResult result, PutMessageResult putMessageResult, - MessageExt messageExt) { + public CompletableFuture submitFlushRequest(AppendMessageResult result, MessageExt messageExt) { // Synchronization flush if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) { final GroupCommitService service = (GroupCommitService) this.flushCommitLogService; @@ -925,8 +932,7 @@ public class CommitLog { } } - public CompletableFuture submitReplicaRequest(AppendMessageResult result, PutMessageResult putMessageResult, - MessageExt messageExt) { + public CompletableFuture submitReplicaRequest(AppendMessageResult result, MessageExt messageExt) { if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) { HAService service = this.defaultMessageStore.getHaService(); if (messageExt.isWaitStoreMsgOK()) { @@ -1419,13 +1425,10 @@ public class CommitLog { for (GroupCommitRequest req : this.requestsRead) { // There may be a message in the next file, so a maximum of // two times the flush - boolean flushOK = false; + boolean flushOK = CommitLog.this.mappedFileQueue.getFlushedWhere() >= req.getNextOffset(); for (int i = 0; i < 2 && !flushOK; i++) { + CommitLog.this.mappedFileQueue.flush(0); flushOK = CommitLog.this.mappedFileQueue.getFlushedWhere() >= req.getNextOffset(); - - if (!flushOK) { - CommitLog.this.mappedFileQueue.flush(0); - } } req.wakeupCustomer(flushOK ? PutMessageStatus.PUT_OK : PutMessageStatus.FLUSH_DISK_TIMEOUT); @@ -1794,6 +1797,11 @@ public class CommitLog { ByteBuffer bornHostHolder = ByteBuffer.allocate(bornHostLength); ByteBuffer storeHostHolder = ByteBuffer.allocate(storeHostLength); + // properties from MessageExtBatch + String batchPropStr = MessageDecoder.messageProperties2String(messageExtBatch.getProperties()); + final byte[] batchPropData = batchPropStr.getBytes(MessageDecoder.CHARSET_UTF8); + final short batchPropLen = (short) batchPropData.length; + while (messagesByteBuff.hasRemaining()) { // 1 TOTALSIZE messagesByteBuff.getInt(); @@ -1817,7 +1825,8 @@ public class CommitLog { final int topicLength = topicData.length; - final int msgLen = calMsgLength(messageExtBatch.getSysFlag(), bodyLen, topicLength, propertiesLen); + final int msgLen = calMsgLength(messageExtBatch.getSysFlag(), bodyLen, topicLength, + propertiesLen + batchPropLen); // Exceeds the maximum message if (msgLen > this.maxMessageSize) { @@ -1870,9 +1879,13 @@ public class CommitLog { this.msgBatchMemory.put((byte) topicLength); this.msgBatchMemory.put(topicData); // 17 PROPERTIES - this.msgBatchMemory.putShort(propertiesLen); - if (propertiesLen > 0) + this.msgBatchMemory.putShort((short) (propertiesLen + batchPropLen)); + if (propertiesLen > 0) { this.msgBatchMemory.put(messagesByteBuff.array(), propertiesPos, propertiesLen); + } + if (batchPropLen > 0) { + this.msgBatchMemory.put(batchPropData, 0, batchPropLen); + } } msgBatchMemory.flip(); return msgBatchMemory; diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 900e9af31fa5d51c8342be4ee8cffa23df49e822..b8ecdee8cc63141876bf6b8f0b2efb3f811479b1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -50,6 +50,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.common.running.RunningStats; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.config.BrokerRole; @@ -391,7 +392,7 @@ public class DefaultMessageStore implements MessageStore { if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) { long value = this.printTimes.getAndIncrement(); if ((value % 50000) == 0) { - log.warn("message store has shutdown, so putMessage is forbidden"); + log.warn("broke role is slave, so putMessage is forbidden"); } return PutMessageStatus.SERVICE_NOT_AVAILABLE; } @@ -399,7 +400,8 @@ public class DefaultMessageStore implements MessageStore { if (!this.runningFlags.isWriteable()) { long value = this.printTimes.getAndIncrement(); if ((value % 50000) == 0) { - log.warn("message store has shutdown, so putMessage is forbidden"); + log.warn("the message store is not writable. It may be caused by one of the following reasons: " + + "the broker's disk is full, write to logic queue error, write to index file error, etc"); } return PutMessageStatus.SERVICE_NOT_AVAILABLE; } else { @@ -803,13 +805,22 @@ public class DefaultMessageStore implements MessageStore { return this.storeStatsService.toString(); } + private String getStorePathPhysic() { + String storePathPhysic = ""; + if (DefaultMessageStore.this.getMessageStoreConfig().isEnableDLegerCommitLog()) { + storePathPhysic = ((DLedgerCommitLog)DefaultMessageStore.this.getCommitLog()).getdLedgerServer().getdLedgerConfig().getDataStorePath(); + } else { + storePathPhysic = DefaultMessageStore.this.getMessageStoreConfig().getStorePathCommitLog(); + } + return storePathPhysic; + } + @Override public HashMap getRuntimeInfo() { HashMap result = this.storeStatsService.getRuntimeInfo(); { - String storePathPhysic = DefaultMessageStore.this.getMessageStoreConfig().getStorePathCommitLog(); - double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic); + double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(getStorePathPhysic()); result.put(RunningStats.commitLogDiskRatio.name(), String.valueOf(physicRatio)); } @@ -1019,7 +1030,8 @@ public class DefaultMessageStore implements MessageStore { Entry> next = it.next(); String topic = next.getKey(); - if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) { + if (!topics.contains(topic) && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC) + && !topic.equals(TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC)) { ConcurrentMap queueTable = next.getValue(); for (ConsumeQueue cq : queueTable.values()) { cq.destroy(); @@ -1050,7 +1062,7 @@ public class DefaultMessageStore implements MessageStore { while (it.hasNext()) { Entry> next = it.next(); String topic = next.getKey(); - if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) { + if (!topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC)) { ConcurrentMap queueTable = next.getValue(); Iterator> itQT = queueTable.entrySet().iterator(); while (itQT.hasNext()) { @@ -1675,8 +1687,7 @@ public class DefaultMessageStore implements MessageStore { cleanImmediately = false; { - String storePathPhysic = DefaultMessageStore.this.getMessageStoreConfig().getStorePathCommitLog(); - double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic); + double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(getStorePathPhysic()); if (physicRatio > diskSpaceWarningLevelRatio) { boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskFull(); if (diskok) { @@ -1736,8 +1747,7 @@ public class DefaultMessageStore implements MessageStore { this.manualDeleteFileSeveralTimes = manualDeleteFileSeveralTimes; } public boolean isSpaceFull() { - String storePathPhysic = DefaultMessageStore.this.getMessageStoreConfig().getStorePathCommitLog(); - double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic); + double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(getStorePathPhysic()); double ratio = DefaultMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0; if (physicRatio > ratio) { DefaultMessageStore.log.info("physic disk of commitLog used: " + physicRatio); @@ -1937,7 +1947,8 @@ public class DefaultMessageStore implements MessageStore { DefaultMessageStore.this.doDispatch(dispatchRequest); if (BrokerRole.SLAVE != DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() - && DefaultMessageStore.this.brokerConfig.isLongPollingEnable()) { + && DefaultMessageStore.this.brokerConfig.isLongPollingEnable() + && DefaultMessageStore.this.messageArrivingListener != null) { DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFile.java b/store/src/main/java/org/apache/rocketmq/store/MappedFile.java index 9185d21d874dce268c937321c2a19ca2a10790eb..b5bdc7766b118f02b7f6c152b85f509ca03e8644 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MappedFile.java +++ b/store/src/main/java/org/apache/rocketmq/store/MappedFile.java @@ -321,7 +321,7 @@ public class MappedFile extends ReferenceResource { int writePos = this.wrotePosition.get(); int lastCommittedPosition = this.committedPosition.get(); - if (writePos - this.committedPosition.get() > 0) { + if (writePos - lastCommittedPosition > commitLeastPages) { try { ByteBuffer byteBuffer = writeBuffer.slice(); byteBuffer.position(lastCommittedPosition); diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 7891f71067a26e4118efde7c79a0b7de7279b8be..848fe22decdb211064003ae1e7c6b7cf832da55d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -54,13 +54,12 @@ public class MessageStoreConfig { /** * introduced since 4.0.x. Determine whether to use mutex reentrantLock when putting message.
- * By default it is set to false indicating using spin lock when putting message. */ - private boolean useReentrantLockWhenPutMessage = false; + private boolean useReentrantLockWhenPutMessage = true; - // Whether schedule flush,default is real-time + // Whether schedule flush @ImportantField - private boolean flushCommitLogTimed = false; + private boolean flushCommitLogTimed = true; // ConsumeQueue flush interval private int flushIntervalConsumeQueue = 1000; // Resource reclaim interval @@ -148,6 +147,12 @@ public class MessageStoreConfig { private String dLegerPeers; private String dLegerSelfId; + private String preferredLeaderId; + + private boolean isEnableBatchPush = false; + + private boolean enableScheduleMessageStats = true; + public boolean isDebugLockEnable() { return debugLockEnable; } @@ -702,4 +707,28 @@ public class MessageStoreConfig { public void setEnableDLegerCommitLog(boolean enableDLegerCommitLog) { this.enableDLegerCommitLog = enableDLegerCommitLog; } + + public String getPreferredLeaderId() { + return preferredLeaderId; + } + + public void setPreferredLeaderId(String preferredLeaderId) { + this.preferredLeaderId = preferredLeaderId; + } + + public boolean isEnableBatchPush() { + return isEnableBatchPush; + } + + public void setEnableBatchPush(boolean enableBatchPush) { + isEnableBatchPush = enableBatchPush; + } + + public boolean isEnableScheduleMessageStats() { + return enableScheduleMessageStats; + } + + public void setEnableScheduleMessageStats(boolean enableScheduleMessageStats) { + this.enableScheduleMessageStats = enableScheduleMessageStats; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 3d748bf8900b5c92ae35cc8b9513109519ce8a86..90fd6f3f169871fa1d6a226b08a6d0a6a8a901f4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -17,19 +17,25 @@ package org.apache.rocketmq.store.dledger; import io.openmessaging.storage.dledger.AppendFuture; +import io.openmessaging.storage.dledger.BatchAppendFuture; import io.openmessaging.storage.dledger.DLedgerConfig; import io.openmessaging.storage.dledger.DLedgerServer; import io.openmessaging.storage.dledger.entry.DLedgerEntry; import io.openmessaging.storage.dledger.protocol.AppendEntryRequest; import io.openmessaging.storage.dledger.protocol.AppendEntryResponse; +import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest; import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode; import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; import io.openmessaging.storage.dledger.store.file.MmapFile; import io.openmessaging.storage.dledger.store.file.MmapFileList; import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; import io.openmessaging.storage.dledger.utils.DLedgerUtils; +import java.net.Inet6Address; +import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.UtilAll; @@ -38,6 +44,7 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.store.AppendMessageResult; import org.apache.rocketmq.store.AppendMessageStatus; import org.apache.rocketmq.store.CommitLog; @@ -71,6 +78,8 @@ public class DLedgerCommitLog extends CommitLog { private boolean isInrecoveringOldCommitlog = false; + private final StringBuilder msgIdBuilder = new StringBuilder(); + public DLedgerCommitLog(final DefaultMessageStore defaultMessageStore) { super(defaultMessageStore); dLedgerConfig = new DLedgerConfig(); @@ -83,6 +92,9 @@ public class DLedgerCommitLog extends CommitLog { dLedgerConfig.setMappedFileSizeForEntryData(defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog()); dLedgerConfig.setDeleteWhen(defaultMessageStore.getMessageStoreConfig().getDeleteWhen()); dLedgerConfig.setFileReservedHours(defaultMessageStore.getMessageStoreConfig().getFileReservedTime() + 1); + dLedgerConfig.setPreferredLeaderId(defaultMessageStore.getMessageStoreConfig().getPreferredLeaderId()); + dLedgerConfig.setEnableBatchPush(defaultMessageStore.getMessageStoreConfig().isEnableBatchPush()); + id = Integer.valueOf(dLedgerConfig.getSelfId().substring(1)) + 1; dLedgerServer = new DLedgerServer(dLedgerConfig); dLedgerFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore(); @@ -360,21 +372,14 @@ public class DLedgerCommitLog extends CommitLog { return beginTimeInDledgerLock; } - @Override - public PutMessageResult putMessage(final MessageExtBrokerInner msg) { + private void setMessageInfo(MessageExtBrokerInner msg, int tranType) { // Set the storage time msg.setStoreTimestamp(System.currentTimeMillis()); // Set the message body BODY CRC (consider the most appropriate setting // on the client) msg.setBodyCRC(UtilAll.crc32(msg.getBody())); - StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); - - String topic = msg.getTopic(); - int queueId = msg.getQueueId(); - //should be consistent with the old version - final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) { // Delay Delivery @@ -383,8 +388,9 @@ public class DLedgerCommitLog extends CommitLog { msg.setDelayTimeLevel(this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()); } - topic = ScheduleMessageService.SCHEDULE_TOPIC; - queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()); + + String topic = TopicValidator.RMQ_SYS_SCHEDULE_TOPIC; + int queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()); // Backup real topic, queueId MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic()); @@ -396,25 +402,46 @@ public class DLedgerCommitLog extends CommitLog { } } + InetSocketAddress bornSocketAddress = (InetSocketAddress) msg.getBornHost(); + if (bornSocketAddress.getAddress() instanceof Inet6Address) { + msg.setBornHostV6Flag(); + } + + InetSocketAddress storeSocketAddress = (InetSocketAddress) msg.getStoreHost(); + if (storeSocketAddress.getAddress() instanceof Inet6Address) { + msg.setStoreHostAddressV6Flag(); + } + } + + @Override + public PutMessageResult putMessage(final MessageExtBrokerInner msg) { + + StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); + final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); + String topic = msg.getTopic(); + setMessageInfo(msg,tranType); + // Back to Results AppendMessageResult appendResult; AppendFuture dledgerFuture; EncodeResult encodeResult; + encodeResult = this.messageSerializer.serialize(msg); + if (encodeResult.status != AppendMessageStatus.PUT_OK) { + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)); + } + putMessageLock.lock(); //spin or ReentrantLock ,depending on store config long elapsedTimeInLock; long queueOffset; try { beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); - encodeResult = this.messageSerializer.serialize(msg); - queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey); - if (encodeResult.status != AppendMessageStatus.PUT_OK) { - return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)); - } + queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); + encodeResult.setQueueOffsetKey(queueOffset, false); AppendEntryRequest request = new AppendEntryRequest(); request.setGroup(dLedgerConfig.getGroup()); request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); - request.setBody(encodeResult.data); + request.setBody(encodeResult.getData()); dledgerFuture = (AppendFuture) dLedgerServer.handleAppend(request); if (dledgerFuture.getPos() == -1) { return new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)); @@ -426,7 +453,7 @@ public class DLedgerCommitLog extends CommitLog { String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; - appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.data.length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock); + appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.getData().length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock); switch (tranType) { case MessageSysFlag.TRANSACTION_PREPARED_TYPE: case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: @@ -487,17 +514,354 @@ public class DLedgerCommitLog extends CommitLog { @Override public PutMessageResult putMessages(final MessageExtBatch messageExtBatch) { - return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); + final int tranType = MessageSysFlag.getTransactionValue(messageExtBatch.getSysFlag()); + + if (tranType != MessageSysFlag.TRANSACTION_NOT_TYPE) { + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); + } + if (messageExtBatch.getDelayTimeLevel() > 0) { + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); + } + + // Set the storage time + messageExtBatch.setStoreTimestamp(System.currentTimeMillis()); + + StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); + + InetSocketAddress bornSocketAddress = (InetSocketAddress) messageExtBatch.getBornHost(); + if (bornSocketAddress.getAddress() instanceof Inet6Address) { + messageExtBatch.setBornHostV6Flag(); + } + + InetSocketAddress storeSocketAddress = (InetSocketAddress) messageExtBatch.getStoreHost(); + if (storeSocketAddress.getAddress() instanceof Inet6Address) { + messageExtBatch.setStoreHostAddressV6Flag(); + } + + // Back to Results + AppendMessageResult appendResult; + BatchAppendFuture dledgerFuture; + EncodeResult encodeResult; + + encodeResult = this.messageSerializer.serialize(messageExtBatch); + if (encodeResult.status != AppendMessageStatus.PUT_OK) { + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult + .status)); + } + + putMessageLock.lock(); //spin or ReentrantLock ,depending on store config + msgIdBuilder.setLength(0); + long elapsedTimeInLock; + long queueOffset; + int msgNum = 0; + try { + beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); + queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); + encodeResult.setQueueOffsetKey(queueOffset, true); + BatchAppendEntryRequest request = new BatchAppendEntryRequest(); + request.setGroup(dLedgerConfig.getGroup()); + request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); + request.setBatchMsgs(encodeResult.batchData); + dledgerFuture = (BatchAppendFuture) dLedgerServer.handleAppend(request); + if (dledgerFuture.getPos() == -1) { + log.warn("HandleAppend return false due to error code {}", dledgerFuture.get().getCode()); + return new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)); + } + long wroteOffset = 0; + + int msgIdLength = (messageExtBatch.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8; + ByteBuffer buffer = ByteBuffer.allocate(msgIdLength); + + boolean isFirstOffset = true; + long firstWroteOffset = 0; + for (long pos : dledgerFuture.getPositions()) { + wroteOffset = pos + DLedgerEntry.BODY_OFFSET; + if (isFirstOffset) { + firstWroteOffset = wroteOffset; + isFirstOffset = false; + } + String msgId = MessageDecoder.createMessageId(buffer, messageExtBatch.getStoreHostBytes(), wroteOffset); + if (msgIdBuilder.length() > 0) { + msgIdBuilder.append(',').append(msgId); + } else { + msgIdBuilder.append(msgId); + } + msgNum++; + } + + elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; + appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen, + msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock); + appendResult.setMsgNum(msgNum); + DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + msgNum); + } catch (Exception e) { + log.error("Put message error", e); + return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus + .UNKNOWN_ERROR)); + } finally { + beginTimeInDledgerLock = 0; + putMessageLock.unlock(); + } + + if (elapsedTimeInLock > 500) { + log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", + elapsedTimeInLock, messageExtBatch.getBody().length, appendResult); + } + + PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR; + try { + AppendEntryResponse appendEntryResponse = dledgerFuture.get(3, TimeUnit.SECONDS); + switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) { + case SUCCESS: + putMessageStatus = PutMessageStatus.PUT_OK; + break; + case INCONSISTENT_LEADER: + case NOT_LEADER: + case LEADER_NOT_READY: + case DISK_FULL: + putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE; + break; + case WAIT_QUORUM_ACK_TIMEOUT: + //Do not return flush_slave_timeout to the client, for the ons client will ignore it. + putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; + break; + case LEADER_PENDING_FULL: + putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; + break; + } + } catch (Throwable t) { + log.error("Failed to get dledger append result", t); + } + + PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult); + if (putMessageStatus == PutMessageStatus.PUT_OK) { + // Statistics + storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).addAndGet(msgNum); + storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).addAndGet(encodeResult.totalMsgLen); + } + return putMessageResult; } @Override public CompletableFuture asyncPutMessage(MessageExtBrokerInner msg) { - return CompletableFuture.completedFuture(this.putMessage(msg)); + + StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); + + final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); + + setMessageInfo(msg, tranType); + + final String finalTopic = msg.getTopic(); + + // Back to Results + AppendMessageResult appendResult; + AppendFuture dledgerFuture; + EncodeResult encodeResult; + + encodeResult = this.messageSerializer.serialize(msg); + if (encodeResult.status != AppendMessageStatus.PUT_OK) { + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status))); + } + putMessageLock.lock(); //spin or ReentrantLock ,depending on store config + long elapsedTimeInLock; + long queueOffset; + try { + beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); + queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); + encodeResult.setQueueOffsetKey(queueOffset, false); + AppendEntryRequest request = new AppendEntryRequest(); + request.setGroup(dLedgerConfig.getGroup()); + request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); + request.setBody(encodeResult.getData()); + dledgerFuture = (AppendFuture) dLedgerServer.handleAppend(request); + if (dledgerFuture.getPos() == -1) { + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } + long wroteOffset = dledgerFuture.getPos() + DLedgerEntry.BODY_OFFSET; + + int msgIdLength = (msg.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8; + ByteBuffer buffer = ByteBuffer.allocate(msgIdLength); + + String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); + elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; + appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.getData().length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock); + switch (tranType) { + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + break; + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + // The next update ConsumeQueue information + DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + 1); + break; + default: + break; + } + } catch (Exception e) { + log.error("Put message error", e); + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } finally { + beginTimeInDledgerLock = 0; + putMessageLock.unlock(); + } + + if (elapsedTimeInLock > 500) { + log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", elapsedTimeInLock, msg.getBody().length, appendResult); + } + + return dledgerFuture.thenApply(appendEntryResponse -> { + PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR; + switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) { + case SUCCESS: + putMessageStatus = PutMessageStatus.PUT_OK; + break; + case INCONSISTENT_LEADER: + case NOT_LEADER: + case LEADER_NOT_READY: + case DISK_FULL: + putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE; + break; + case WAIT_QUORUM_ACK_TIMEOUT: + //Do not return flush_slave_timeout to the client, for the ons client will ignore it. + putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; + break; + case LEADER_PENDING_FULL: + putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; + break; + } + PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult); + if (putMessageStatus == PutMessageStatus.PUT_OK) { + // Statistics + storeStatsService.getSinglePutMessageTopicTimesTotal(finalTopic).incrementAndGet(); + storeStatsService.getSinglePutMessageTopicSizeTotal(msg.getTopic()).addAndGet(appendResult.getWroteBytes()); + } + return putMessageResult; + }); } @Override public CompletableFuture asyncPutMessages(MessageExtBatch messageExtBatch) { - return CompletableFuture.completedFuture(putMessages(messageExtBatch)); + final int tranType = MessageSysFlag.getTransactionValue(messageExtBatch.getSysFlag()); + + if (tranType != MessageSysFlag.TRANSACTION_NOT_TYPE) { + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null)); + } + if (messageExtBatch.getDelayTimeLevel() > 0) { + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null)); + } + + // Set the storage time + messageExtBatch.setStoreTimestamp(System.currentTimeMillis()); + + StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); + + InetSocketAddress bornSocketAddress = (InetSocketAddress) messageExtBatch.getBornHost(); + if (bornSocketAddress.getAddress() instanceof Inet6Address) { + messageExtBatch.setBornHostV6Flag(); + } + + InetSocketAddress storeSocketAddress = (InetSocketAddress) messageExtBatch.getStoreHost(); + if (storeSocketAddress.getAddress() instanceof Inet6Address) { + messageExtBatch.setStoreHostAddressV6Flag(); + } + + // Back to Results + AppendMessageResult appendResult; + BatchAppendFuture dledgerFuture; + EncodeResult encodeResult; + + encodeResult = this.messageSerializer.serialize(messageExtBatch); + if (encodeResult.status != AppendMessageStatus.PUT_OK) { + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult + .status))); + } + + putMessageLock.lock(); //spin or ReentrantLock ,depending on store config + msgIdBuilder.setLength(0); + long elapsedTimeInLock; + long queueOffset; + int msgNum = 0; + try { + beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); + queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); + encodeResult.setQueueOffsetKey(queueOffset, true); + BatchAppendEntryRequest request = new BatchAppendEntryRequest(); + request.setGroup(dLedgerConfig.getGroup()); + request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); + request.setBatchMsgs(encodeResult.batchData); + dledgerFuture = (BatchAppendFuture) dLedgerServer.handleAppend(request); + if (dledgerFuture.getPos() == -1) { + log.warn("HandleAppend return false due to error code {}", dledgerFuture.get().getCode()); + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } + long wroteOffset = 0; + + int msgIdLength = (messageExtBatch.getSysFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 + 8 : 16 + 4 + 8; + ByteBuffer buffer = ByteBuffer.allocate(msgIdLength); + + boolean isFirstOffset = true; + long firstWroteOffset = 0; + for (long pos : dledgerFuture.getPositions()) { + wroteOffset = pos + DLedgerEntry.BODY_OFFSET; + if (isFirstOffset) { + firstWroteOffset = wroteOffset; + isFirstOffset = false; + } + String msgId = MessageDecoder.createMessageId(buffer, messageExtBatch.getStoreHostBytes(), wroteOffset); + if (msgIdBuilder.length() > 0) { + msgIdBuilder.append(',').append(msgId); + } else { + msgIdBuilder.append(msgId); + } + msgNum++; + } + + elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; + appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen, + msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock); + appendResult.setMsgNum(msgNum); + DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + msgNum); + } catch (Exception e) { + log.error("Put message error", e); + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } finally { + beginTimeInDledgerLock = 0; + putMessageLock.unlock(); + } + + if (elapsedTimeInLock > 500) { + log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", + elapsedTimeInLock, messageExtBatch.getBody().length, appendResult); + } + + return dledgerFuture.thenApply(appendEntryResponse -> { + PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR; + switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) { + case SUCCESS: + putMessageStatus = PutMessageStatus.PUT_OK; + break; + case INCONSISTENT_LEADER: + case NOT_LEADER: + case LEADER_NOT_READY: + case DISK_FULL: + putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE; + break; + case WAIT_QUORUM_ACK_TIMEOUT: + //Do not return flush_slave_timeout to the client, for the ons client will ignore it. + putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; + break; + case LEADER_PENDING_FULL: + putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; + break; + } + PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult); + if (putMessageStatus == PutMessageStatus.PUT_OK) { + // Statistics + storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).incrementAndGet(); + storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).addAndGet(appendResult.getWroteBytes()); + } + return putMessageResult; + }); } @Override @@ -562,51 +926,83 @@ public class DLedgerCommitLog extends CommitLog { return diff; } + private long getQueueOffsetByKey(String key, int tranType) { + Long queueOffset = DLedgerCommitLog.this.topicQueueTable.get(key); + if (null == queueOffset) { + queueOffset = 0L; + DLedgerCommitLog.this.topicQueueTable.put(key, queueOffset); + } + + // Transaction messages that require special handling + switch (tranType) { + // Prepared and Rollback message is not consumed, will not enter the + // consumer queuec + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + queueOffset = 0L; + break; + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + default: + break; + } + return queueOffset; + } + + class EncodeResult { private String queueOffsetKey; - private byte[] data; + private ByteBuffer data; + private List batchData; private AppendMessageStatus status; + private int totalMsgLen; - public EncodeResult(AppendMessageStatus status, byte[] data, String queueOffsetKey) { + public EncodeResult(AppendMessageStatus status, ByteBuffer data, String queueOffsetKey) { this.data = data; this.status = status; this.queueOffsetKey = queueOffsetKey; } + + public void setQueueOffsetKey(long offset, boolean isBatch) { + if (!isBatch) { + this.data.putLong(MessageDecoder.QUEUE_OFFSET_POSITION, offset); + return; + } + + for (byte[] data : batchData) { + ByteBuffer.wrap(data).putLong(MessageDecoder.QUEUE_OFFSET_POSITION, offset++); + } + } + + public byte[] getData() { + return data.array(); + } + + public EncodeResult(AppendMessageStatus status, String queueOffsetKey, List batchData, int totalMsgLen) { + this.batchData = batchData; + this.status = status; + this.queueOffsetKey = queueOffsetKey; + this.totalMsgLen = totalMsgLen; + } } class MessageSerializer { - // File at the end of the minimum fixed length empty - private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4; - private final ByteBuffer msgIdMemory; - private final ByteBuffer msgIdV6Memory; - // Store the message content - private final ByteBuffer msgStoreItemMemory; + // The maximum length of the message private final int maxMessageSize; - // Build Message Key - private final StringBuilder keyBuilder = new StringBuilder(); - - private final StringBuilder msgIdBuilder = new StringBuilder(); - -// private final ByteBuffer hostHolder = ByteBuffer.allocate(8); MessageSerializer(final int size) { - this.msgIdMemory = ByteBuffer.allocate(4 + 4 + 8); - this.msgIdV6Memory = ByteBuffer.allocate(16 + 4 + 8); - this.msgStoreItemMemory = ByteBuffer.allocate(size + END_FILE_MIN_BLANK_LENGTH); this.maxMessageSize = size; } - public ByteBuffer getMsgStoreItemMemory() { - return msgStoreItemMemory; - } - public EncodeResult serialize(final MessageExtBrokerInner msgInner) { // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET
// PHY OFFSET long wroteOffset = 0; + long queueOffset = 0; + int sysflag = msgInner.getSysFlag(); int bornHostLength = (sysflag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; @@ -614,33 +1010,7 @@ public class DLedgerCommitLog extends CommitLog { ByteBuffer bornHostHolder = ByteBuffer.allocate(bornHostLength); ByteBuffer storeHostHolder = ByteBuffer.allocate(storeHostLength); - // Record ConsumeQueue information - keyBuilder.setLength(0); - keyBuilder.append(msgInner.getTopic()); - keyBuilder.append('-'); - keyBuilder.append(msgInner.getQueueId()); - String key = keyBuilder.toString(); - - Long queueOffset = DLedgerCommitLog.this.topicQueueTable.get(key); - if (null == queueOffset) { - queueOffset = 0L; - DLedgerCommitLog.this.topicQueueTable.put(key, queueOffset); - } - - // Transaction messages that require special handling - final int tranType = MessageSysFlag.getTransactionValue(msgInner.getSysFlag()); - switch (tranType) { - // Prepared and Rollback message is not consumed, will not enter the - // consumer queuec - case MessageSysFlag.TRANSACTION_PREPARED_TYPE: - case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: - queueOffset = 0L; - break; - case MessageSysFlag.TRANSACTION_NOT_TYPE: - case MessageSysFlag.TRANSACTION_COMMIT_TYPE: - default: - break; - } + String key = msgInner.getTopic() + "-" + msgInner.getQueueId(); /** * Serialize message @@ -662,6 +1032,8 @@ public class DLedgerCommitLog extends CommitLog { final int msgLen = calMsgLength(msgInner.getSysFlag(), bodyLength, topicLength, propertiesLength); + ByteBuffer msgStoreItemMemory = ByteBuffer.allocate(msgLen); + // Exceeds the maximum message if (msgLen > this.maxMessageSize) { DLedgerCommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength @@ -671,60 +1043,163 @@ public class DLedgerCommitLog extends CommitLog { // Initialization of storage space this.resetByteBuffer(msgStoreItemMemory, msgLen); // 1 TOTALSIZE - this.msgStoreItemMemory.putInt(msgLen); + msgStoreItemMemory.putInt(msgLen); // 2 MAGICCODE - this.msgStoreItemMemory.putInt(DLedgerCommitLog.MESSAGE_MAGIC_CODE); + msgStoreItemMemory.putInt(DLedgerCommitLog.MESSAGE_MAGIC_CODE); // 3 BODYCRC - this.msgStoreItemMemory.putInt(msgInner.getBodyCRC()); + msgStoreItemMemory.putInt(msgInner.getBodyCRC()); // 4 QUEUEID - this.msgStoreItemMemory.putInt(msgInner.getQueueId()); + msgStoreItemMemory.putInt(msgInner.getQueueId()); // 5 FLAG - this.msgStoreItemMemory.putInt(msgInner.getFlag()); + msgStoreItemMemory.putInt(msgInner.getFlag()); // 6 QUEUEOFFSET - this.msgStoreItemMemory.putLong(queueOffset); + msgStoreItemMemory.putLong(queueOffset); // 7 PHYSICALOFFSET - this.msgStoreItemMemory.putLong(wroteOffset); + msgStoreItemMemory.putLong(wroteOffset); // 8 SYSFLAG - this.msgStoreItemMemory.putInt(msgInner.getSysFlag()); + msgStoreItemMemory.putInt(msgInner.getSysFlag()); // 9 BORNTIMESTAMP - this.msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); + msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); // 10 BORNHOST - this.resetByteBuffer(bornHostHolder, bornHostLength); - this.msgStoreItemMemory.put(msgInner.getBornHostBytes(bornHostHolder)); + resetByteBuffer(bornHostHolder, bornHostLength); + msgStoreItemMemory.put(msgInner.getBornHostBytes(bornHostHolder)); // 11 STORETIMESTAMP - this.msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); + msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); // 12 STOREHOSTADDRESS - this.resetByteBuffer(storeHostHolder, storeHostLength); - this.msgStoreItemMemory.put(msgInner.getStoreHostBytes(storeHostHolder)); + resetByteBuffer(storeHostHolder, storeHostLength); + msgStoreItemMemory.put(msgInner.getStoreHostBytes(storeHostHolder)); //this.msgBatchMemory.put(msgInner.getStoreHostBytes()); // 13 RECONSUMETIMES - this.msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); + msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); // 14 Prepared Transaction Offset - this.msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset()); + msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset()); // 15 BODY - this.msgStoreItemMemory.putInt(bodyLength); + msgStoreItemMemory.putInt(bodyLength); if (bodyLength > 0) { - this.msgStoreItemMemory.put(msgInner.getBody()); + msgStoreItemMemory.put(msgInner.getBody()); } // 16 TOPIC - this.msgStoreItemMemory.put((byte) topicLength); - this.msgStoreItemMemory.put(topicData); + msgStoreItemMemory.put((byte) topicLength); + msgStoreItemMemory.put(topicData); // 17 PROPERTIES - this.msgStoreItemMemory.putShort((short) propertiesLength); + msgStoreItemMemory.putShort((short) propertiesLength); if (propertiesLength > 0) { - this.msgStoreItemMemory.put(propertiesData); + msgStoreItemMemory.put(propertiesData); + } + return new EncodeResult(AppendMessageStatus.PUT_OK, msgStoreItemMemory, key); + } + + public EncodeResult serialize(final MessageExtBatch messageExtBatch) { + String key = messageExtBatch.getTopic() + "-" + messageExtBatch.getQueueId(); + + int totalMsgLen = 0; + ByteBuffer messagesByteBuff = messageExtBatch.wrap(); + List batchBody = new LinkedList<>(); + + int sysFlag = messageExtBatch.getSysFlag(); + int bornHostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; + int storeHostLength = (sysFlag & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; + ByteBuffer bornHostHolder = ByteBuffer.allocate(bornHostLength); + ByteBuffer storeHostHolder = ByteBuffer.allocate(storeHostLength); + + while (messagesByteBuff.hasRemaining()) { + // 1 TOTALSIZE + messagesByteBuff.getInt(); + // 2 MAGICCODE + messagesByteBuff.getInt(); + // 3 BODYCRC + messagesByteBuff.getInt(); + // 4 FLAG + int flag = messagesByteBuff.getInt(); + // 5 BODY + int bodyLen = messagesByteBuff.getInt(); + int bodyPos = messagesByteBuff.position(); + int bodyCrc = UtilAll.crc32(messagesByteBuff.array(), bodyPos, bodyLen); + messagesByteBuff.position(bodyPos + bodyLen); + // 6 properties + short propertiesLen = messagesByteBuff.getShort(); + int propertiesPos = messagesByteBuff.position(); + messagesByteBuff.position(propertiesPos + propertiesLen); + + final byte[] topicData = messageExtBatch.getTopic().getBytes(MessageDecoder.CHARSET_UTF8); + + final int topicLength = topicData.length; + + final int msgLen = calMsgLength(messageExtBatch.getSysFlag(), bodyLen, topicLength, propertiesLen); + ByteBuffer msgStoreItemMemory = ByteBuffer.allocate(msgLen); + + // Exceeds the maximum message + if (msgLen > this.maxMessageSize) { + CommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + + bodyLen + + ", maxMessageSize: " + this.maxMessageSize); + throw new RuntimeException("message size exceeded"); + } + + totalMsgLen += msgLen; + // Determines whether there is sufficient free space + if (totalMsgLen > maxMessageSize) { + throw new RuntimeException("message size exceeded"); + } + + // Initialization of storage space + this.resetByteBuffer(msgStoreItemMemory, msgLen); + // 1 TOTALSIZE + msgStoreItemMemory.putInt(msgLen); + // 2 MAGICCODE + msgStoreItemMemory.putInt(DLedgerCommitLog.MESSAGE_MAGIC_CODE); + // 3 BODYCRC + msgStoreItemMemory.putInt(bodyCrc); + // 4 QUEUEID + msgStoreItemMemory.putInt(messageExtBatch.getQueueId()); + // 5 FLAG + msgStoreItemMemory.putInt(flag); + // 6 QUEUEOFFSET + msgStoreItemMemory.putLong(0L); + // 7 PHYSICALOFFSET + msgStoreItemMemory.putLong(0); + // 8 SYSFLAG + msgStoreItemMemory.putInt(messageExtBatch.getSysFlag()); + // 9 BORNTIMESTAMP + msgStoreItemMemory.putLong(messageExtBatch.getBornTimestamp()); + // 10 BORNHOST + resetByteBuffer(bornHostHolder, bornHostLength); + msgStoreItemMemory.put(messageExtBatch.getBornHostBytes(bornHostHolder)); + // 11 STORETIMESTAMP + msgStoreItemMemory.putLong(messageExtBatch.getStoreTimestamp()); + // 12 STOREHOSTADDRESS + resetByteBuffer(storeHostHolder, storeHostLength); + msgStoreItemMemory.put(messageExtBatch.getStoreHostBytes(storeHostHolder)); + // 13 RECONSUMETIMES + msgStoreItemMemory.putInt(messageExtBatch.getReconsumeTimes()); + // 14 Prepared Transaction Offset + msgStoreItemMemory.putLong(0); + // 15 BODY + msgStoreItemMemory.putInt(bodyLen); + if (bodyLen > 0) { + msgStoreItemMemory.put(messagesByteBuff.array(), bodyPos, bodyLen); + } + // 16 TOPIC + msgStoreItemMemory.put((byte) topicLength); + msgStoreItemMemory.put(topicData); + // 17 PROPERTIES + msgStoreItemMemory.putShort(propertiesLen); + if (propertiesLen > 0) { + msgStoreItemMemory.put(messagesByteBuff.array(), propertiesPos, propertiesLen); + } + byte[] data = new byte[msgLen]; + msgStoreItemMemory.clear(); + msgStoreItemMemory.get(data); + batchBody.add(data); } - byte[] data = new byte[msgLen]; - this.msgStoreItemMemory.clear(); - this.msgStoreItemMemory.get(data); - return new EncodeResult(AppendMessageStatus.PUT_OK, data, key); + + return new EncodeResult(AppendMessageStatus.PUT_OK, key, batchBody, totalMsgLen); } private void resetByteBuffer(final ByteBuffer byteBuffer, final int limit) { byteBuffer.flip(); byteBuffer.limit(limit); } - } public static class DLedgerSelectMappedBufferResult extends SelectMappedBufferResult { @@ -736,6 +1211,7 @@ public class DLedgerCommitLog extends CommitLog { this.sbr = sbr; } + @Override public synchronized void release() { super.release(); if (sbr != null) { diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java index a0e77fa3dc96ea0b5b23a405a2f0f05701d5b41d..34c51eb9e26fc5df4efbc8bef58d360f759a5fe5 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java @@ -590,6 +590,13 @@ public class HAService { log.info(this.getServiceName() + " service end"); } + + @Override + public void shutdown() { + super.shutdown(); + closeMaster(); + } + // private void disableWriteFlag() { // if (this.socketChannel != null) { // SelectionKey sk = this.socketChannel.keyFor(this.selector); diff --git a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java index 3be8cbc809ed59f6d39de94cd407a01741a899be..bacae1e80bcfb6d7455a88ed493efb270cd595d1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java +++ b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java @@ -25,9 +25,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.common.ConfigManager; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageAccessor; @@ -48,7 +48,6 @@ import org.apache.rocketmq.store.config.StorePathConfigHelper; public class ScheduleMessageService extends ConfigManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); - public static final String SCHEDULE_TOPIC = "SCHEDULE_TOPIC_XXXX"; private static final long FIRST_DELAY_TIME = 1000L; private static final long DELAY_FOR_A_WHILE = 100L; private static final long DELAY_FOR_A_PERIOD = 10000L; @@ -91,7 +90,7 @@ public class ScheduleMessageService extends ConfigManager { Map.Entry next = it.next(); int queueId = delayLevel2QueueId(next.getKey()); long delayOffset = next.getValue(); - long maxOffset = this.defaultMessageStore.getMaxOffsetInQueue(SCHEDULE_TOPIC, queueId); + long maxOffset = this.defaultMessageStore.getMaxOffsetInQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, queueId); String value = String.format("%d,%d", delayOffset, maxOffset); String key = String.format("%s_%d", RunningStats.scheduleMessageOffset.name(), next.getKey()); stats.put(key, value); @@ -113,6 +112,7 @@ public class ScheduleMessageService extends ConfigManager { public void start() { if (started.compareAndSet(false, true)) { + super.load(); this.timer = new Timer("ScheduleMessageTimerThread", true); for (Map.Entry entry : this.delayLevelTable.entrySet()) { Integer level = entry.getKey(); @@ -262,7 +262,7 @@ public class ScheduleMessageService extends ConfigManager { public void executeOnTimeup() { ConsumeQueue cq = - ScheduleMessageService.this.defaultMessageStore.findConsumeQueue(SCHEDULE_TOPIC, + ScheduleMessageService.this.defaultMessageStore.findConsumeQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, delayLevel2QueueId(delayLevel)); long failScheduleOffset = offset; @@ -306,7 +306,7 @@ public class ScheduleMessageService extends ConfigManager { if (msgExt != null) { try { MessageExtBrokerInner msgInner = this.messageTimeup(msgExt); - if (MixAll.RMQ_SYS_TRANS_HALF_TOPIC.equals(msgInner.getTopic())) { + if (TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC.equals(msgInner.getTopic())) { log.error("[BUG] the real topic of schedule msg is {}, discard the msg. msg={}", msgInner.getTopic(), msgInner); continue; @@ -317,6 +317,12 @@ public class ScheduleMessageService extends ConfigManager { if (putMessageResult != null && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { + if (ScheduleMessageService.this.defaultMessageStore.getMessageStoreConfig().isEnableScheduleMessageStats()) { + ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); + ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), + putMessageResult.getAppendMessageResult().getWroteBytes()); + ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incBrokerPutNums(putMessageResult.getAppendMessageResult().getMsgNum()); + } continue; } else { // XXX: warn and notify me diff --git a/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java b/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java index 8618dbb0e56ed2804303d1b08bbe94cc5e54f500..b3a7c196f94996e3d967d2efaad89a416f3eeaaa 100644 --- a/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/BatchPutMessageTest.java @@ -35,6 +35,7 @@ import java.io.File; import java.net.InetSocketAddress; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -80,6 +81,12 @@ public class BatchPutMessageTest { @Test public void testPutMessages() throws Exception { + String batchPropK = "extraKey"; + String batchPropV = "extraValue"; + Map batchProp = new HashMap<>(1); + batchProp.put(batchPropK, batchPropV); + short batchPropLen = (short) messageProperties2String(batchProp).getBytes(MessageDecoder.CHARSET_UTF8).length; + List messages = new ArrayList<>(); String topic = "batch-write-topic"; int queue = 0; @@ -98,7 +105,7 @@ public class BatchPutMessageTest { short propertiesLength = (short) propertiesBytes.length; final byte[] topicData = msg.getTopic().getBytes(MessageDecoder.CHARSET_UTF8); final int topicLength = topicData.length; - msgLengthArr[j] = calMsgLength(msg.getBody().length, topicLength, propertiesLength) + msgLengthArr[j - 1]; + msgLengthArr[j] = calMsgLength(msg.getBody().length, topicLength, propertiesLength+batchPropLen) + msgLengthArr[j - 1]; j++; } byte[] batchMessageBody = MessageDecoder.encodeMessages(messages); @@ -106,6 +113,7 @@ public class BatchPutMessageTest { messageExtBatch.setTopic(topic); messageExtBatch.setQueueId(queue); messageExtBatch.setBody(batchMessageBody); + messageExtBatch.putUserProperty(batchPropK,batchPropV); messageExtBatch.setBornTimestamp(System.currentTimeMillis()); messageExtBatch.setStoreHost(new InetSocketAddress("127.0.0.1", 125)); messageExtBatch.setBornHost(new InetSocketAddress("127.0.0.1", 126)); diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java index e43bb4bc3d3f84ab382fee0adb492082c37a8222..db7d367e02135cdb577a4f97624cf93cfd69be22 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java @@ -40,10 +40,11 @@ public class DefaultMessageStoreShutDownTest { @Before public void init() throws Exception { - messageStore = spy(buildMessageStore()); - boolean load = messageStore.load(); + DefaultMessageStore store = buildMessageStore(); + boolean load = store.load(); assertTrue(load); - messageStore.start(); + store.start(); + messageStore = spy(store); when(messageStore.dispatchBehindBytes()).thenReturn(100L); } diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java index fd13e71a427d07db60400392a8718bcc64de0298..440f9146314a049a91e7096f4b382399b878f8d6 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java @@ -133,7 +133,7 @@ public class DefaultMessageStoreTest { StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore); for (long i = 0; i < totalMsgs; i++) { - GetMessageResult result = messageStore.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); + GetMessageResult result = messageStore.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null); assertThat(result).isNotNull(); result.release(); } @@ -471,7 +471,7 @@ public class DefaultMessageStoreTest { StoreTestUtil.waitCommitLogReput((DefaultMessageStore) messageStore); for (long i = 0; i < totalMsgs; i++) { - GetMessageResult result = master.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); + GetMessageResult result = master.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null); assertThat(result).isNotNull(); result.release(); diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java index a736754de47b4ce6d699b63360dfebf1eb86ee0d..5660de1363d71da69d7ceaf088cc2210df8a3655 100644 --- a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java @@ -16,17 +16,19 @@ */ package org.apache.rocketmq.store; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExtBatch; +import org.junit.After; + import java.io.File; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.UnknownHostException; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.rocketmq.common.UtilAll; -import org.junit.After; public class StoreTestBase { @@ -44,6 +46,28 @@ public class StoreTestBase { return port.addAndGet(5); } + protected MessageExtBatch buildBatchMessage(int size) { + MessageExtBatch messageExtBatch = new MessageExtBatch(); + messageExtBatch.setTopic("StoreTest"); + messageExtBatch.setTags("TAG1"); + messageExtBatch.setKeys("Hello"); + messageExtBatch.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + messageExtBatch.setSysFlag(0); + + messageExtBatch.setBornTimestamp(System.currentTimeMillis()); + messageExtBatch.setBornHost(BornHost); + messageExtBatch.setStoreHost(StoreHost); + + List messageList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + messageList.add(buildMessage()); + } + + messageExtBatch.setBody(MessageDecoder.encodeMessages(messageList)); + + return messageExtBatch; + } + protected MessageExtBrokerInner buildMessage() { MessageExtBrokerInner msg = new MessageExtBrokerInner(); msg.setTopic("StoreTest"); @@ -59,6 +83,40 @@ public class StoreTestBase { return msg; } + protected MessageExtBatch buildIPv6HostBatchMessage(int size) { + MessageExtBatch messageExtBatch = new MessageExtBatch(); + messageExtBatch.setTopic("StoreTest"); + messageExtBatch.setTags("TAG1"); + messageExtBatch.setKeys("Hello"); + messageExtBatch.setBody(MessageBody); + messageExtBatch.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); + messageExtBatch.setKeys(String.valueOf(System.currentTimeMillis())); + messageExtBatch.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + messageExtBatch.setSysFlag(0); + messageExtBatch.setBornHostV6Flag(); + messageExtBatch.setStoreHostAddressV6Flag(); + messageExtBatch.setBornTimestamp(System.currentTimeMillis()); + try { + messageExtBatch.setBornHost(new InetSocketAddress(InetAddress.getByName("1050:0000:0000:0000:0005:0600:300c:326b"), 8123)); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + + try { + messageExtBatch.setStoreHost(new InetSocketAddress(InetAddress.getByName("::1"), 8123)); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + + List messageList = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + messageList.add(buildIPv6HostMessage()); + } + + messageExtBatch.setBody(MessageDecoder.encodeMessages(messageList)); + return messageExtBatch; + } + protected MessageExtBrokerInner buildIPv6HostMessage() { MessageExtBrokerInner msg = new MessageExtBrokerInner(); msg.setTopic("StoreTest"); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index f0b9205302e965a832393d788d122e64bbb5d0aa..8ab8a23b4715ec4ff0742b2f5b0df4a6239c9f02 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -19,12 +19,17 @@ package org.apache.rocketmq.store.dledger; import io.openmessaging.storage.dledger.DLedgerServer; import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; import io.openmessaging.storage.dledger.store.file.MmapFileList; + import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; @@ -39,7 +44,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { @Test public void testTruncateCQ() throws Exception { - String base = createBaseDir(); + String base = createBaseDir(); String peers = String.format("n0-localhost:%d", nextPort()); String group = UUID.randomUUID().toString(); String topic = UUID.randomUUID().toString(); @@ -92,10 +97,9 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { } - @Test public void testRecover() throws Exception { - String base = createBaseDir(); + String base = createBaseDir(); String peers = String.format("n0-localhost:%d", nextPort()); String group = UUID.randomUUID().toString(); String topic = UUID.randomUUID().toString(); @@ -133,10 +137,9 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { } - @Test public void testPutAndGetMessage() throws Exception { - String base = createBaseDir(); + String base = createBaseDir(); String peers = String.format("n0-localhost:%d", nextPort()); String group = UUID.randomUUID().toString(); DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0); @@ -146,7 +149,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { List results = new ArrayList<>(); for (int i = 0; i < 10; i++) { MessageExtBrokerInner msgInner = - i < 5 ? buildMessage() : buildIPv6HostMessage(); + i < 5 ? buildMessage() : buildIPv6HostMessage(); msgInner.setTopic(topic); msgInner.setQueueId(0); PutMessageResult putMessageResult = messageStore.putMessage(msgInner); @@ -158,7 +161,93 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); Assert.assertEquals(10, messageStore.getMaxOffsetInQueue(topic, 0)); Assert.assertEquals(0, messageStore.dispatchBehindBytes()); - GetMessageResult getMessageResult = messageStore.getMessage("group", topic, 0, 0, 32, null); + GetMessageResult getMessageResult = messageStore.getMessage("group", topic, 0, 0, 32, null); + Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus()); + + Assert.assertEquals(10, getMessageResult.getMessageBufferList().size()); + Assert.assertEquals(10, getMessageResult.getMessageMapedList().size()); + + for (int i = 0; i < results.size(); i++) { + ByteBuffer buffer = getMessageResult.getMessageBufferList().get(i); + MessageExt messageExt = MessageDecoder.decode(buffer); + Assert.assertEquals(i, messageExt.getQueueOffset()); + Assert.assertEquals(results.get(i).getAppendMessageResult().getMsgId(), messageExt.getMsgId()); + Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset()); + } + messageStore.destroy(); + messageStore.shutdown(); + } + + @Test + public void testBatchPutAndGetMessage() throws Exception { + String base = createBaseDir(); + String peers = String.format("n0-localhost:%d", nextPort()); + String group = UUID.randomUUID().toString(); + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0); + Thread.sleep(1000); + String topic = UUID.randomUUID().toString(); + // should be less than 4 + int batchMessageSize = 2; + int repeat = 10; + List results = new ArrayList<>(); + for (int i = 0; i < repeat; i++) { + MessageExtBatch messageExtBatch = + i < repeat / 10 ? buildBatchMessage(batchMessageSize) : buildIPv6HostBatchMessage(batchMessageSize); + messageExtBatch.setTopic(topic); + messageExtBatch.setQueueId(0); + PutMessageResult putMessageResult = messageStore.putMessages(messageExtBatch); + results.add(putMessageResult); + Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus()); + Assert.assertEquals(i * batchMessageSize, putMessageResult.getAppendMessageResult().getLogicsOffset()); + } + Thread.sleep(100); + Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(repeat * batchMessageSize, messageStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + GetMessageResult getMessageResult = messageStore.getMessage("group", topic, 0, 0, 100, null); + Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus()); + + Assert.assertEquals(repeat * batchMessageSize > 32 ? 32 : repeat * batchMessageSize, getMessageResult.getMessageBufferList().size()); + Assert.assertEquals(repeat * batchMessageSize > 32 ? 32 : repeat * batchMessageSize, getMessageResult.getMessageMapedList().size()); + Assert.assertEquals(repeat * batchMessageSize, getMessageResult.getMaxOffset()); + + for (int i = 0; i < results.size(); i++) { + ByteBuffer buffer = getMessageResult.getMessageBufferList().get(i * batchMessageSize); + MessageExt messageExt = MessageDecoder.decode(buffer); + Assert.assertEquals(i * batchMessageSize, messageExt.getQueueOffset()); + Assert.assertEquals(results.get(i).getAppendMessageResult().getMsgId().split(",").length, batchMessageSize); + Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset()); + } + messageStore.destroy(); + messageStore.shutdown(); + } + + @Test + public void testAsyncPutAndGetMessage() throws Exception { + String base = createBaseDir(); + String peers = String.format("n0-localhost:%d", nextPort()); + String group = UUID.randomUUID().toString(); + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0); + Thread.sleep(1000); + String topic = UUID.randomUUID().toString(); + + List results = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + MessageExtBrokerInner msgInner = + i < 5 ? buildMessage() : buildIPv6HostMessage(); + msgInner.setTopic(topic); + msgInner.setQueueId(0); + CompletableFuture futureResult = messageStore.asyncPutMessage(msgInner); + PutMessageResult putMessageResult = futureResult.get(3000, TimeUnit.MILLISECONDS); + results.add(putMessageResult); + Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus()); + Assert.assertEquals(i, putMessageResult.getAppendMessageResult().getLogicsOffset()); + } + Thread.sleep(100); + Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(10, messageStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + GetMessageResult getMessageResult = messageStore.getMessage("group", topic, 0, 0, 32, null); Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus()); Assert.assertEquals(10, getMessageResult.getMessageBufferList().size()); @@ -175,15 +264,60 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { messageStore.shutdown(); } + @Test + public void testAsyncBatchPutAndGetMessage() throws Exception { + String base = createBaseDir(); + String peers = String.format("n0-localhost:%d", nextPort()); + String group = UUID.randomUUID().toString(); + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0); + Thread.sleep(1000); + String topic = UUID.randomUUID().toString(); + // should be less than 4 + int batchMessageSize = 2; + int repeat = 10; + + List results = new ArrayList<>(); + for (int i = 0; i < repeat; i++) { + MessageExtBatch messageExtBatch = + i < 5 ? buildBatchMessage(batchMessageSize) : buildIPv6HostBatchMessage(batchMessageSize); + messageExtBatch.setTopic(topic); + messageExtBatch.setQueueId(0); + CompletableFuture futureResult = messageStore.asyncPutMessages(messageExtBatch); + PutMessageResult putMessageResult = futureResult.get(3000, TimeUnit.MILLISECONDS); + results.add(putMessageResult); + Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus()); + Assert.assertEquals(i * batchMessageSize, putMessageResult.getAppendMessageResult().getLogicsOffset()); + } + Thread.sleep(100); + Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(repeat * batchMessageSize, messageStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + GetMessageResult getMessageResult = messageStore.getMessage("group", topic, 0, 0, 32, null); + Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus()); + + Assert.assertEquals(repeat * batchMessageSize > 32 ? 32 : repeat * batchMessageSize, getMessageResult.getMessageBufferList().size()); + Assert.assertEquals(repeat * batchMessageSize > 32 ? 32 : repeat * batchMessageSize, getMessageResult.getMessageMapedList().size()); + Assert.assertEquals(repeat * batchMessageSize, getMessageResult.getMaxOffset()); + + for (int i = 0; i < results.size(); i++) { + ByteBuffer buffer = getMessageResult.getMessageBufferList().get(i * batchMessageSize); + MessageExt messageExt = MessageDecoder.decode(buffer); + Assert.assertEquals(i * batchMessageSize, messageExt.getQueueOffset()); + Assert.assertEquals(results.get(i).getAppendMessageResult().getMsgId().split(",").length, batchMessageSize); + Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset()); + } + messageStore.destroy(); + messageStore.shutdown(); + } @Test public void testCommittedPos() throws Exception { String peers = String.format("n0-localhost:%d;n1-localhost:%d", nextPort(), nextPort()); String group = UUID.randomUUID().toString(); - DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group,"n0", peers, "n0", false, 0); + DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group, "n0", peers, "n0", false, 0); String topic = UUID.randomUUID().toString(); - MessageExtBrokerInner msgInner = buildMessage(); + MessageExtBrokerInner msgInner = buildMessage(); msgInner.setTopic(topic); msgInner.setQueueId(0); PutMessageResult putMessageResult = leaderStore.putMessage(msgInner); @@ -195,7 +329,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0)); - DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group,"n1", peers, "n0", false, 0); + DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group, "n1", peers, "n0", false, 0); Thread.sleep(2000); Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0)); @@ -214,10 +348,10 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { public void testIPv6HostMsgCommittedPos() throws Exception { String peers = String.format("n0-localhost:%d;n1-localhost:%d", nextPort(), nextPort()); String group = UUID.randomUUID().toString(); - DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group,"n0", peers, "n0", false, 0); + DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group, "n0", peers, "n0", false, 0); String topic = UUID.randomUUID().toString(); - MessageExtBrokerInner msgInner = buildIPv6HostMessage(); + MessageExtBrokerInner msgInner = buildIPv6HostMessage(); msgInner.setTopic(topic); msgInner.setQueueId(0); PutMessageResult putMessageResult = leaderStore.putMessage(msgInner); @@ -229,7 +363,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0)); - DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group,"n1", peers, "n0", false, 0); + DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group, "n1", peers, "n0", false, 0); Thread.sleep(2000); Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0)); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java index 5b0ca347579dc7e391dab25645d5a87620a89269..5864b282c7bbff80bc4ca7a2e4db54541ace3bbd 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -63,9 +63,9 @@ public class MessageStoreTestBase extends StoreTestBase { if (leaderId != null) { dLegerServer.getdLedgerConfig().setEnableLeaderElector(false); if (selfId.equals(leaderId)) { - dLegerServer.getMemberState().changeToLeader(-1); + dLegerServer.getMemberState().changeToLeader(0); } else { - dLegerServer.getMemberState().changeToFollower(-1, leaderId); + dLegerServer.getMemberState().changeToFollower(0, leaderId); } } diff --git a/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java b/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java index fd860e6b9d7ea5d8a3d2c6e3a4fee714cda35d57..fa3c6bfcd8b9fd08035d5c7c409a10337f32b8d0 100644 --- a/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java @@ -40,6 +40,9 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; +import static org.apache.rocketmq.store.stats.BrokerStatsManager.BROKER_PUT_NUMS; +import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_NUMS; +import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_SIZE; import static org.assertj.core.api.Assertions.assertThat; @@ -112,6 +115,10 @@ public class ScheduleMessageServiceTest { @Test public void deliverDelayedMessageTimerTaskTest() throws Exception { + assertThat(messageStore.getMessageStoreConfig().isEnableScheduleMessageStats()).isTrue(); + + assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_NUMS, topic)).isNull(); + MessageExtBrokerInner msg = buildMessage(); int realQueueId = msg.getQueueId(); // set delayLevel,and send delay message @@ -141,6 +148,10 @@ public class ScheduleMessageServiceTest { // now,found the message assertThat(messageResult.getStatus()).isEqualTo(GetMessageStatus.FOUND); + // get the stats change + assertThat(messageStore.getBrokerStatsManager().getStatsItem(BROKER_PUT_NUMS, brokerConfig.getBrokerClusterName()).getValue().get()).isEqualTo(1); + assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_NUMS, topic).getValue().get()).isEqualTo(1L); + assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_SIZE, topic).getValue().get()).isEqualTo(messageResult.getBufferTotalSize()); // get the message body ByteBuffer byteBuffer = ByteBuffer.allocate(messageResult.getBufferTotalSize()); diff --git a/test/pom.xml b/test/pom.xml index ce9a8a9a7dff4240fc52019d3453e12937799808..41296501c28b649e170660c8490325641be836ce 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -31,7 +31,6 @@ log4j log4j - 1.2.17 ${project.groupId} diff --git a/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java index 44db782b5c53b82ece5db8fba626071f6f8407fb..e08967eb91de53826980b94f530c3e2ff299ae6c 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java @@ -33,7 +33,7 @@ public class FileUtil { this.fileName = fileName; } - public static void main(String args[]) { + public static void main(String[] args) { String filePath = FileUtil.class.getResource("/").getPath(); String fileName = "test.txt"; FileUtil fileUtil = new FileUtil(filePath, fileName); diff --git a/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java b/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java index 6edeecadb2a2d2ec19b55a14801f49bf0321409b..0c24427d9248788a4db636a14d114d6ff1f66196 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java @@ -76,7 +76,7 @@ public class MQWait { } } - public static void main(String args[]) { + public static void main(String[] args) { long start = System.currentTimeMillis(); MQWait.setCondition(new Condition() { diff --git a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java index 1c2bdac305781e3923bdb3bc334f03548f26639c..41cacb684c3c175fa8316a3e2ce10dc2dd546389 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java @@ -100,14 +100,14 @@ public final class RandomUtil { return n + res % (m - n); } - private static char getChar(int arg[]) { + private static char getChar(int[] arg) { int size = arg.length; int c = rd.nextInt(size / 2); c = c * 2; return (char) (getIntegerBetween(arg[c], arg[c + 1])); } - private static String getString(int n, int arg[]) { + private static String getString(int n, int[] arg) { StringBuilder res = new StringBuilder(); for (int i = 0; i < n; i++) { res.append(getChar(arg)); @@ -116,17 +116,17 @@ public final class RandomUtil { } public static String getStringWithCharacter(int n) { - int arg[] = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; + int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; return getString(n, arg); } public static String getStringWithNumber(int n) { - int arg[] = new int[] {'0', '9' + 1}; + int[] arg = new int[] {'0', '9' + 1}; return getString(n, arg); } public static String getStringWithNumAndCha(int n) { - int arg[] = new int[] {'a', 'z' + 1, 'A', 'Z' + 1, '0', '9' + 1}; + int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1, '0', '9' + 1}; return getString(n, arg); } diff --git a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java index 9eca28bbe7d90c8e0bf5f9e15525ce943735c9b8..3f71176d6f01db16000a49cf6fc47e5c6e57ce9c 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java @@ -45,16 +45,16 @@ public class RandomUtils { } public static String getStringWithNumber(int n) { - int arg[] = new int[] {'0', '9' + 1}; + int[] arg = new int[] {'0', '9' + 1}; return getString(n, arg); } public static String getStringWithCharacter(int n) { - int arg[] = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; + int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; return getString(n, arg); } - private static String getString(int n, int arg[]) { + private static String getString(int n, int[] arg) { StringBuilder res = new StringBuilder(); for (int i = 0; i < n; i++) { res.append(getChar(arg)); @@ -62,7 +62,7 @@ public class RandomUtils { return res.toString(); } - private static char getChar(int arg[]) { + private static char getChar(int[] arg) { int size = arg.length; int c = rd.nextInt(size / 2); c = c * 2; diff --git a/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java b/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java index 965d2ee6705c467d7cde0bdcd77c06a57c814fff..69bfd8f7d10ed5a07b24621cab801819f3805697 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java @@ -140,7 +140,7 @@ public class VerifyUtils { return rtExpect; } - public static void main(String args[]) { + public static void main(String[] args) { verifyBalance(400, 0.1f, 230, 190); } } diff --git a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java index 82ab461aa53607c475f1aff7f6adda1118be728f..bdd991a335fcd170da3c604dfa1101c5c326ee86 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java @@ -43,7 +43,7 @@ public class ListDataCollectorImpl implements DataCollector { return datas; } - public void resetData() { + public synchronized void resetData() { datas.clear(); unlockIncrement(); } @@ -67,7 +67,7 @@ public class ListDataCollectorImpl implements DataCollector { return Collections.frequency(datas, data) == 1; } - public Collection getAllDataWithoutDuplicate() { + public synchronized Collection getAllDataWithoutDuplicate() { return new HashSet(datas); } @@ -81,7 +81,7 @@ public class ListDataCollectorImpl implements DataCollector { return res; } - public void removeData(Object data) { + public synchronized void removeData(Object data) { datas.remove(data); } diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java index 564da5ceaf1cf701ea91639f6fb780de126bc76b..0edbdbe2e386c2bea5029ce571387ec794a22315 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java @@ -92,7 +92,7 @@ public class MulTagSubIT extends BaseConf { @Test public void testSubTwoTabAndMatchTwo() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; String subExpress = String.format("%s||%s", tags[0], tags[1]); int msgSize = 10; @@ -113,7 +113,7 @@ public class MulTagSubIT extends BaseConf { @Test public void testSubThreeTabAndMatchTwo() { - String tags[] = {"jueyin1", "jueyin2", "jueyin3"}; + String[] tags = {"jueyin1", "jueyin2", "jueyin3"}; String subExpress = String.format("%s||%s", tags[0], tags[1]); int msgSize = 10; @@ -135,7 +135,7 @@ public class MulTagSubIT extends BaseConf { @Test public void testNoMatch() { - String tags[] = {"jueyin1", "jueyin2", "jueyin3"}; + String[] tags = {"jueyin1", "jueyin2", "jueyin3"}; String subExpress = "no_match"; int msgSize = 10; diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java index 31321975b57b9c2c0e52096a9d0a6bd113b4ed0f..8de1b7d4e0110919151ae7d0d567aeb430fad5ff 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java @@ -84,7 +84,7 @@ public class TagMessageWithMulConsumerIT extends BaseConf { @Test public void testSendMessagesWithTwoTag() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; int msgSize = 10; TagMessage tagMessage = new TagMessage(tags, topic, msgSize); @@ -113,7 +113,7 @@ public class TagMessageWithMulConsumerIT extends BaseConf { @Test public void testTwoConsumerOneMatchOneOtherMatchAll() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; String sub1 = String.format("%s||%s", tags[0], tags[1]); String sub2 = String.format("%s|| noExist", tags[0]); int msgSize = 10; @@ -144,7 +144,7 @@ public class TagMessageWithMulConsumerIT extends BaseConf { @Test public void testSubKindsOf() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; String sub1 = String.format("%s||%s", tags[0], tags[1]); String sub2 = String.format("%s|| noExist", tags[0]); String sub3 = tags[0]; diff --git a/test/src/test/resources/log4j.xml b/test/src/test/resources/log4j.xml index 3031095e70521e65a472f18c392d957e3981d35a..7840ab78c110eb870b06b901b75d2094027a1444 100644 --- a/test/src/test/resources/log4j.xml +++ b/test/src/test/resources/log4j.xml @@ -30,7 +30,7 @@ - + diff --git a/tools/pom.xml b/tools/pom.xml index c6938afdb7e188a6b4a5a6a76619aea090eddbc0..abe8197de3974b6c3b1f52f9e5efbbf5078224dd 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.7.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -56,10 +56,6 @@ ch.qos.logback logback-classic - - ch.qos.logback - logback-core - org.apache.commons commons-lang3 diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index 1ca3fe4c2f482a286b485021544d74dca16d39bf..6592639035f922a362f7503342d34f7ff06ac11b 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -26,7 +26,6 @@ import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.AclConfig; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.admin.ConsumeStats; @@ -51,6 +50,7 @@ import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingConnectException; @@ -62,7 +62,7 @@ import org.apache.rocketmq.tools.admin.api.MessageTrack; public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { private final DefaultMQAdminExtImpl defaultMQAdminExtImpl; private String adminExtGroup = "admin_ext_group"; - private String createTopicKey = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC; + private String createTopicKey = TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC; private long timeoutMillis = 5000; public DefaultMQAdminExt() { @@ -128,12 +128,18 @@ public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { } @Override - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, - long end) throws MQClientException, - InterruptedException { + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return defaultMQAdminExtImpl.queryMessage(topic, key, maxNum, begin, end); } + public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + + return defaultMQAdminExtImpl.queryMessageByUniqKey(topic, key, maxNum, begin, end); + } + @Override public void start() throws MQClientException { defaultMQAdminExtImpl.start(); @@ -320,6 +326,13 @@ public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { defaultMQAdminExtImpl.deleteSubscriptionGroup(addr, groupName); } + @Override + public void deleteSubscriptionGroup(String addr, + String groupName, boolean removeOffset) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + defaultMQAdminExtImpl.deleteSubscriptionGroup(addr, groupName, removeOffset); + } + @Override public void createAndUpdateKvConfig(String namespace, String key, String value) throws RemotingException, MQBrokerException, diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index 22d4005ce9a9e031c0b10e1d09ae6b1a471cc65c..8930bbe49d207ab2dc361caaee6d88f75762cbea 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -424,7 +424,14 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { public void deleteSubscriptionGroup(String addr, String groupName) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - this.mqClientInstance.getMQClientAPIImpl().deleteSubscriptionGroup(addr, groupName, timeoutMillis); + this.mqClientInstance.getMQClientAPIImpl().deleteSubscriptionGroup(addr, groupName, false, timeoutMillis); + } + + @Override + public void deleteSubscriptionGroup(String addr, + String groupName, boolean removeOffset) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + this.mqClientInstance.getMQClientAPIImpl().deleteSubscriptionGroup(addr, groupName, removeOffset, timeoutMillis); } @Override @@ -991,12 +998,18 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { } @Override - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, - long end) throws MQClientException, - InterruptedException { + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.mqClientInstance.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); } + public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, + long end) throws MQClientException, InterruptedException { + + return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(topic, key, maxNum, begin, end); + } + @Override public void updateConsumeOffset(String brokerAddr, String consumeGroup, MessageQueue mq, long offset) throws RemotingException, InterruptedException, MQBrokerException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java index 17b62251c60d5af461db085bb8587fbcf9f14f4f..d5462cb04e5a41873690c10cf67b880e21ad6bd1 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java @@ -152,6 +152,9 @@ public interface MQAdminExt extends MQAdmin { void deleteSubscriptionGroup(final String addr, String groupName) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + void deleteSubscriptionGroup(final String addr, String groupName, boolean removeOffset) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + void createAndUpdateKvConfig(String namespace, String key, String value) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index 28431a963ee676ea5acef4c9ebd0ee8b9e0e2f2e..f9477445bea3d895dd9f52a0c7f7a1127ee95d67 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -59,6 +59,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByIdSubCommand; import org.apache.rocketmq.tools.command.message.QueryMsgByKeySubCommand; import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; +import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; import org.apache.rocketmq.tools.command.message.SendMessageCommand; import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; @@ -164,6 +165,7 @@ public class MQAdminStartup { initCommand(new QueryMsgByKeySubCommand()); initCommand(new QueryMsgByUniqueKeySubCommand()); initCommand(new QueryMsgByOffsetSubCommand()); + initCommand(new QueryMsgTraceByIdSubCommand()); initCommand(new PrintMessageSubCommand()); initCommand(new PrintMessageByQueueCommand()); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java index 11a36048328f4e40ebcfc0decb3092cd19f4238c..11346bf6f7f18f132913aab90b959f3c0a21a3bf 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java @@ -48,11 +48,11 @@ public class GetBrokerConfigCommand implements SubCommand { @Override public Options buildCommandlineOptions(final Options options) { - Option opt = new Option("b", "brokerAddr", true, "update which broker"); + Option opt = new Option("b", "brokerAddr", true, "get which broker"); opt.setRequired(false); options.addOption(opt); - opt = new Option("c", "clusterName", true, "update which cluster"); + opt = new Option("c", "clusterName", true, "get which cluster"); opt.setRequired(false); options.addOption(opt); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java index 5038123561bf0aafb880ee0100d3e3323c2c49ae..872a130d9f52ccafd240b49c3b32850e2b56f505 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java @@ -38,7 +38,7 @@ import org.apache.rocketmq.tools.command.SubCommandException; public class CLusterSendMsgRTCommand implements SubCommand { - public static void main(String args[]) { + public static void main(String[] args) { } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java index b946ee141ebc507da3df465b9a9fa31c043790a7..7985f9d5918cd918e52d84f0f954d04709158c83 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java @@ -111,17 +111,26 @@ public class ConsumerProgressSubCommand implements SubCommand { if (showClientIP) { messageQueueAllocationResult = getMessageQueueAllocationResult(defaultMQAdminExt, consumerGroup); } - - System.out.printf("%-32s %-32s %-4s %-20s %-20s %-20s %-20s %s%n", - "#Topic", - "#Broker Name", - "#QID", - "#Broker Offset", - "#Consumer Offset", - "#Client IP", - "#Diff", - "#LastTime"); - + if (showClientIP) { + System.out.printf("%-32s %-32s %-4s %-20s %-20s %-20s %-20s %s%n", + "#Topic", + "#Broker Name", + "#QID", + "#Broker Offset", + "#Consumer Offset", + "#Client IP", + "#Diff", + "#LastTime"); + } else { + System.out.printf("%-32s %-32s %-4s %-20s %-20s %-20s %s%n", + "#Topic", + "#Broker Name", + "#QID", + "#Broker Offset", + "#Consumer Offset", + "#Diff", + "#LastTime"); + } long diffTotal = 0L; for (MessageQueue mq : mqList) { OffsetWrapper offsetWrapper = consumeStats.getOffsetTable().get(mq); @@ -141,17 +150,28 @@ public class ConsumerProgressSubCommand implements SubCommand { if (showClientIP) { clientIP = messageQueueAllocationResult.get(mq); } - - System.out.printf("%-32s %-32s %-4d %-20d %-20d %-20s %-20d %s%n", - UtilAll.frontStringAtLeast(mq.getTopic(), 32), - UtilAll.frontStringAtLeast(mq.getBrokerName(), 32), - mq.getQueueId(), - offsetWrapper.getBrokerOffset(), - offsetWrapper.getConsumerOffset(), - null != clientIP ? clientIP : "N/A", - diff, - lastTime - ); + if (showClientIP) { + System.out.printf("%-32s %-32s %-4d %-20d %-20d %-20s %-20d %s%n", + UtilAll.frontStringAtLeast(mq.getTopic(), 32), + UtilAll.frontStringAtLeast(mq.getBrokerName(), 32), + mq.getQueueId(), + offsetWrapper.getBrokerOffset(), + offsetWrapper.getConsumerOffset(), + null != clientIP ? clientIP : "N/A", + diff, + lastTime + ); + } else { + System.out.printf("%-32s %-32s %-4d %-20d %-20d %-20d %s%n", + UtilAll.frontStringAtLeast(mq.getTopic(), 32), + UtilAll.frontStringAtLeast(mq.getBrokerName(), 32), + mq.getQueueId(), + offsetWrapper.getBrokerOffset(), + offsetWrapper.getConsumerOffset(), + diff, + lastTime + ); + } } System.out.printf("%n"); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java index 96d81956e4540e4574a027ebf9a88ffd16284431..fb0efebaa215445a52c016f5a90abb5cf06e56db 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java @@ -54,6 +54,10 @@ public class DeleteSubscriptionGroupCommand implements SubCommand { opt.setRequired(true); options.addOption(opt); + opt = new Option("r", "removeOffset", true, "remove offset"); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -65,11 +69,16 @@ public class DeleteSubscriptionGroupCommand implements SubCommand { // groupName String groupName = commandLine.getOptionValue('g').trim(); + boolean removeOffset = false; + if (commandLine.hasOption('r')) { + removeOffset = Boolean.valueOf(commandLine.getOptionValue("r").trim()); + } + if (commandLine.hasOption('b')) { String addr = commandLine.getOptionValue('b').trim(); adminExt.start(); - adminExt.deleteSubscriptionGroup(addr, groupName); + adminExt.deleteSubscriptionGroup(addr, groupName, removeOffset); System.out.printf("delete subscription group [%s] from broker [%s] success.%n", groupName, addr); @@ -80,7 +89,7 @@ public class DeleteSubscriptionGroupCommand implements SubCommand { Set masterSet = CommandUtil.fetchMasterAddrByClusterName(adminExt, clusterName); for (String master : masterSet) { - adminExt.deleteSubscriptionGroup(master, groupName); + adminExt.deleteSubscriptionGroup(master, groupName, removeOffset); System.out.printf( "delete subscription group [%s] from broker [%s] in cluster [%s] success.%n", groupName, master, clusterName); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java index 9ad075082f667823e779c7a5413a0bc9d0e2171e..8c7bec616d7068a8fab2c8c03d79cabcfb1e64cb 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.UtilAll; @@ -57,82 +58,43 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { } } - public static void queryById(final DefaultMQAdminExt admin, final String topic, - final String msgId) throws MQClientException, - RemotingException, MQBrokerException, InterruptedException, IOException { - MessageExt msg = admin.viewMessage(topic, msgId); - - String bodyTmpFilePath = createBodyFile(msg); - - System.out.printf("%-20s %s%n", - "Topic:", - msg.getTopic() - ); - - System.out.printf("%-20s %s%n", - "Tags:", - "[" + msg.getTags() + "]" - ); - - System.out.printf("%-20s %s%n", - "Keys:", - "[" + msg.getKeys() + "]" - ); - - System.out.printf("%-20s %d%n", - "Queue ID:", - msg.getQueueId() - ); - - System.out.printf("%-20s %d%n", - "Queue Offset:", - msg.getQueueOffset() - ); - - System.out.printf("%-20s %d%n", - "CommitLog Offset:", - msg.getCommitLogOffset() - ); - - System.out.printf("%-20s %d%n", - "Reconsume Times:", - msg.getReconsumeTimes() - ); - - System.out.printf("%-20s %s%n", - "Born Timestamp:", - UtilAll.timeMillisToHumanString2(msg.getBornTimestamp()) - ); - - System.out.printf("%-20s %s%n", - "Store Timestamp:", - UtilAll.timeMillisToHumanString2(msg.getStoreTimestamp()) - ); - - System.out.printf("%-20s %s%n", - "Born Host:", - RemotingHelper.parseSocketAddressAddr(msg.getBornHost()) - ); - - System.out.printf("%-20s %s%n", - "Store Host:", - RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()) - ); - - System.out.printf("%-20s %d%n", - "System Flag:", - msg.getSysFlag() - ); - - System.out.printf("%-20s %s%n", - "Properties:", - msg.getProperties() != null ? msg.getProperties().toString() : "" - ); - - System.out.printf("%-20s %s%n", - "Message Body Path:", - bodyTmpFilePath - ); + public static void queryById(final DefaultMQAdminExt admin, final String topic, final String msgId, + final boolean showAll) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException, IOException { + + QueryResult queryResult = admin.queryMessageByUniqKey(topic, msgId, 32, 0, Long.MAX_VALUE); + assert queryResult != null; + List list = queryResult.getMessageList(); + if (list == null || list.size() == 0) { + return; + } + list.sort((o1, o2) -> (int) (o1.getStoreTimestamp() - o2.getStoreTimestamp())); + for (int i = 0; i < (showAll ? list.size() : 1); i++) { + showMessage(admin, list.get(i), i); + } + } + + private static void showMessage(final DefaultMQAdminExt admin, MessageExt msg, int index) throws IOException { + String bodyTmpFilePath = createBodyFile(msg, index); + + final String strFormat = "%-20s %s%n"; + final String intFormat = "%-20s %d%n"; + + System.out.printf(strFormat, "Topic:", msg.getTopic()); + System.out.printf(strFormat, "Tags:", "[" + msg.getTags() + "]"); + System.out.printf(strFormat, "Keys:", "[" + msg.getKeys() + "]"); + System.out.printf(intFormat, "Queue ID:", msg.getQueueId()); + System.out.printf(intFormat, "Queue Offset:", msg.getQueueOffset()); + System.out.printf(intFormat, "CommitLog Offset:", msg.getCommitLogOffset()); + System.out.printf(intFormat, "Reconsume Times:", msg.getReconsumeTimes()); + System.out.printf(strFormat, "Born Timestamp:", UtilAll.timeMillisToHumanString2(msg.getBornTimestamp())); + System.out.printf(strFormat, "Store Timestamp:", UtilAll.timeMillisToHumanString2(msg.getStoreTimestamp())); + System.out.printf(strFormat, "Born Host:", RemotingHelper.parseSocketAddressAddr(msg.getBornHost())); + System.out.printf(strFormat, "Store Host:", RemotingHelper.parseSocketAddressAddr(msg.getStoreHost())); + System.out.printf(intFormat, "System Flag:", msg.getSysFlag()); + System.out.printf(strFormat, "Properties:", + msg.getProperties() != null ? msg.getProperties().toString() : ""); + System.out.printf(strFormat, "Message Body Path:", bodyTmpFilePath); try { List mtdList = admin.messageTrackDetail(msg); @@ -149,18 +111,21 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { } } - private static String createBodyFile(MessageExt msg) throws IOException { + private static String createBodyFile(MessageExt msg, int index) throws IOException { DataOutputStream dos = null; try { - String bodyTmpFilePath = "/tmp/rocketmq/msgbodys"; - File file = new File(bodyTmpFilePath); + StringBuffer bodyTmpFilePath = new StringBuffer("/tmp/rocketmq/msgbodys"); + File file = new File(bodyTmpFilePath.toString()); if (!file.exists()) { file.mkdirs(); } - bodyTmpFilePath = bodyTmpFilePath + "/" + msg.getMsgId(); - dos = new DataOutputStream(new FileOutputStream(bodyTmpFilePath)); + bodyTmpFilePath.append("/").append(msg.getMsgId()); + if (index > 0) { + bodyTmpFilePath.append("_" + index); + } + dos = new DataOutputStream(new FileOutputStream(bodyTmpFilePath.toString())); dos.write(msg.getBody()); - return bodyTmpFilePath; + return bodyTmpFilePath.toString(); } finally { if (dos != null) dos.close(); @@ -195,6 +160,10 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { opt.setRequired(true); options.addOption(opt); + opt = new Option("a", "showAll", false, "Print all message, the limit is 32"); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -202,11 +171,11 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { try { - defaultMQAdminExt = createMQAdminExt(rpcHook); final String msgId = commandLine.getOptionValue('i').trim(); final String topic = commandLine.getOptionValue('t').trim(); + final boolean showAll = commandLine.hasOption('a'); if (commandLine.hasOption('g') && commandLine.hasOption('d')) { final String consumerGroup = commandLine.getOptionValue('g').trim(); final String clientId = commandLine.getOptionValue('d').trim(); @@ -214,7 +183,7 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); System.out.printf("%s", result); } else { - queryById(defaultMQAdminExt, topic, msgId); + queryById(defaultMQAdminExt, topic, msgId, showAll); } } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java new file mode 100644 index 0000000000000000000000000000000000000000..7382ff568dcf04668687acf37eceacbbce1f614f --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.message; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.rocketmq.client.QueryResult; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.trace.TraceType; +import org.apache.rocketmq.client.trace.TraceView; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class QueryMsgTraceByIdSubCommand implements SubCommand { + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("i", "msgId", true, "Message Id"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "traceTopic", true, "The name value of message trace topic"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + @Override + public String commandDesc() { + return "query a message trace"; + } + + @Override + public String commandName() { + return "QueryMsgTraceById"; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + final String msgId = commandLine.getOptionValue('i').trim(); + String traceTopic = TopicValidator.RMQ_SYS_TRACE_TOPIC; + if (commandLine.hasOption('t')) { + traceTopic = commandLine.getOptionValue('t').trim(); + } + this.queryTraceByMsgId(defaultMQAdminExt, traceTopic, msgId); + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + "command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + private void queryTraceByMsgId(final DefaultMQAdminExt admin, String traceTopic, String msgId) + throws MQClientException, InterruptedException { + admin.start(); + QueryResult queryResult = admin.queryMessage(traceTopic, msgId, 64, 0, System.currentTimeMillis()); + List messageList = queryResult.getMessageList(); + List traceViews = new ArrayList<>(); + for (MessageExt message : messageList) { + List traceView = TraceView.decodeFromTraceTransData(msgId, message); + traceViews.addAll(traceView); + } + + this.printMessageTrace(traceViews); + } + + private void printMessageTrace(List traceViews) { + Map> consumerTraceMap = new HashMap<>(16); + for (TraceView traceView : traceViews) { + if (traceView.getMsgType().equals(TraceType.Pub.name())) { + System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", + "#Type", + "#ProducerGroup", + "#ClientHost", + "#SendTime", + "#CostTimes", + "#Status" + ); + System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", + "Pub", + traceView.getGroupName(), + traceView.getClientHost(), + DateFormatUtils.format(traceView.getTimeStamp(), "yyyy-MM-dd HH:mm:ss"), + traceView.getCostTime() + "ms", + traceView.getStatus() + ); + System.out.printf("\n"); + } + if (traceView.getMsgType().equals(TraceType.SubAfter.name())) { + String groupName = traceView.getGroupName(); + if (consumerTraceMap.containsKey(groupName)) { + consumerTraceMap.get(groupName).add(traceView); + } else { + ArrayList views = new ArrayList<>(); + views.add(traceView); + consumerTraceMap.put(groupName, views); + } + } + } + + Iterator consumers = consumerTraceMap.keySet().iterator(); + while (consumers.hasNext()) { + System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", + "#Type", + "#ConsumerGroup", + "#ClientHost", + "#ConsumerTime", + "#CostTimes", + "#Status" + ); + List consumerTraces = consumerTraceMap.get(consumers.next()); + for (TraceView traceView : consumerTraces) { + System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", + "Sub", + traceView.getGroupName(), + traceView.getClientHost(), + DateFormatUtils.format(traceView.getTimeStamp(), "yyyy-MM-dd HH:mm:ss"), + traceView.getCostTime() + "ms", + traceView.getStatus() + ); + } + System.out.printf("\n"); + } + } +} \ No newline at end of file diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java index e4921c6f04ef0839bf60bf68835e17234b2cafae..9550742e8f7fa25c154a844ab49a63de3fe9a82f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java @@ -68,14 +68,18 @@ public class SendMessageCommand implements SubCommand { opt.setRequired(false); options.addOption(opt); + opt = new Option("m", "msgTraceEnable", true, "Message Trace Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + return options; } - private DefaultMQProducer createProducer(RPCHook rpcHook) { + private DefaultMQProducer createProducer(RPCHook rpcHook, boolean msgTraceEnable) { if (this.producer != null) { return producer; } else { - producer = new DefaultMQProducer(rpcHook); + producer = new DefaultMQProducer(null, rpcHook, msgTraceEnable, null); producer.setProducerGroup(Long.toString(System.currentTimeMillis())); return producer; } @@ -112,8 +116,11 @@ public class SendMessageCommand implements SubCommand { } catch (Exception e) { throw new RuntimeException(this.getClass().getSimpleName() + " command failed", e); } - - DefaultMQProducer producer = this.createProducer(rpcHook); + boolean msgTraceEnable = false; + if (commandLine.hasOption('m')) { + msgTraceEnable = Boolean.parseBoolean(commandLine.getOptionValue('m').trim()); + } + DefaultMQProducer producer = this.createProducer(rpcHook, msgTraceEnable); SendResult result; try { producer.start(); @@ -153,4 +160,4 @@ public class SendMessageCommand implements SubCommand { ); } } -} \ No newline at end of file +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java index c770db01d427c9e33c17a8a6cc20a76ed6760dfc..a78a4a63ab0eb6495561eaf734f06cffd8a627c7 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java @@ -19,14 +19,23 @@ package org.apache.rocketmq.tools.command.topic; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + public class TopicRouteSubCommand implements SubCommand { + private static final String FORMAT = "%-45s %-32s %-50s %-10s %-11s %-5s%n"; + @Override public String commandName() { return "topicRoute"; @@ -43,6 +52,9 @@ public class TopicRouteSubCommand implements SubCommand { opt.setRequired(true); options.addOption(opt); + opt = new Option("l", "list", false, "Use list format to print data"); + opt.setRequired(false); + options.addOption(opt); return options; } @@ -58,12 +70,46 @@ public class TopicRouteSubCommand implements SubCommand { String topic = commandLine.getOptionValue('t').trim(); TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); - String json = topicRouteData.toJson(true); - System.out.printf("%s%n", json); + printData(topicRouteData, commandLine.hasOption('l')); } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); } finally { defaultMQAdminExt.shutdown(); } } -} + + private void printData(TopicRouteData topicRouteData, boolean useListFormat) { + if (!useListFormat) { + System.out.printf("%s%n", topicRouteData.toJson(true)); + return; + } + + int totalReadQueue = 0, totalWriteQueue = 0; + List queueDataList = topicRouteData.getQueueDatas(); + Map map = new HashMap<>(); + for (QueueData queueData : queueDataList) { + map.put(queueData.getBrokerName(), queueData); + } + queueDataList.sort(Comparator.comparing(QueueData::getBrokerName)); + + List brokerDataList = topicRouteData.getBrokerDatas(); + brokerDataList.sort(Comparator.comparing(BrokerData::getBrokerName)); + + System.out.printf(FORMAT, "#ClusterName", "#BrokerName", "#BrokerAddrs", "#ReadQueue", "#WriteQueue", "#Perm"); + + for (BrokerData brokerData : brokerDataList) { + String brokerName = brokerData.getBrokerName(); + QueueData queueData = map.get(brokerName); + totalReadQueue += queueData.getReadQueueNums(); + totalWriteQueue += queueData.getWriteQueueNums(); + System.out.printf(FORMAT, brokerData.getCluster(), brokerName, brokerData.getBrokerAddrs(), + queueData.getReadQueueNums(), queueData.getWriteQueueNums(), queueData.getPerm()); + } + + for (int i = 0; i < 158; i++) { + System.out.print("-"); + } + System.out.printf("%n"); + System.out.printf(FORMAT, "Total:", map.keySet().size(), "", totalReadQueue, totalWriteQueue, ""); + } +} \ No newline at end of file diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java index a06a19d1fd08e800d51ffe58bae006b3140ee8df..63a2a19275a65b9078beb23324c61a5260f6b41d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java @@ -94,7 +94,7 @@ public class UpdateTopicPermSubCommand implements SubCommand { topicConfig.setTopicName(topic); topicConfig.setWriteQueueNums(queueData.getWriteQueueNums()); topicConfig.setReadQueueNums(queueData.getReadQueueNums()); - topicConfig.setTopicSysFlag(queueData.getTopicSynFlag()); + topicConfig.setTopicSysFlag(queueData.getTopicSysFlag()); //new perm int perm; if (commandLine.hasOption('p')) { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/monitor/MonitorService.java b/tools/src/main/java/org/apache/rocketmq/tools/monitor/MonitorService.java index 9bf09ad410754be36bb2cb75e01761d83a02288c..94f588da27537575e8e99fc73b5e707425edc417 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/monitor/MonitorService.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/monitor/MonitorService.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.admin.ConsumeStats; import org.apache.rocketmq.common.admin.OffsetWrapper; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; @@ -83,7 +84,7 @@ public class MonitorService { try { this.defaultMQPushConsumer.setConsumeThreadMin(1); this.defaultMQPushConsumer.setConsumeThreadMax(1); - this.defaultMQPushConsumer.subscribe(MixAll.OFFSET_MOVED_EVENT, "*"); + this.defaultMQPushConsumer.subscribe(TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT, "*"); this.defaultMQPushConsumer.registerMessageListener(new MessageListenerConcurrently() { @Override diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommandTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f61c71d109297e31170f3ce414ce55fa72bd6f98 --- /dev/null +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommandTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.message; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExtImpl; +import org.apache.rocketmq.tools.command.SubCommandException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Field; + +import static org.mockito.Mockito.mock; + +public class QueryMsgTraceByIdSubCommandTest { + private static DefaultMQAdminExt defaultMQAdminExt; + private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; + private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); + private static MQClientAPIImpl mQClientAPIImpl; + + @BeforeClass + public static void init() throws NoSuchFieldException, IllegalAccessException, InterruptedException, RemotingTimeoutException, MQClientException, RemotingSendRequestException, RemotingConnectException, MQBrokerException, UnsupportedEncodingException { + mQClientAPIImpl = mock(MQClientAPIImpl.class); + defaultMQAdminExt = new DefaultMQAdminExt(); + defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000); + + Field field = DefaultMQAdminExtImpl.class.getDeclaredField("mqClientInstance"); + field.setAccessible(true); + field.set(defaultMQAdminExtImpl, mqClientInstance); + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mqClientInstance, mQClientAPIImpl); + field = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl"); + field.setAccessible(true); + field.set(defaultMQAdminExt, defaultMQAdminExtImpl); + } + + @AfterClass + public static void terminate() { + defaultMQAdminExt.shutdown(); + } + + @Ignore + @Test + public void testExecute() throws SubCommandException { + System.setProperty("rocketmq.namesrv.addr", "127.0.0.1:9876"); + QueryMsgTraceByIdSubCommand cmd = new QueryMsgTraceByIdSubCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = new String[] {"-i AC1FF54E81C418B4AAC24F92E1E00000"}; + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, cmd.buildCommandlineOptions(options), new PosixParser()); + cmd.execute(commandLine, options, null); + } +} \ No newline at end of file