diff --git a/acl/pom.xml b/acl/pom.xml index 36dae6ad045b67762f4221ab9df54e90666f8b7a..d62493417892e67b78a46f414f95320be263b6a7 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT rocketmq-acl rocketmq-acl ${project.version} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java index bfe96f53037cca259b837a461114f879b2b7359c..d129c66d1c81dbb0cbce7dc7d5324eacdd53864d 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java @@ -44,6 +44,16 @@ public class AclConstants { public static final String CONFIG_TIME_STAMP = "timestamp"; + public static final String PUB = "PUB"; + + public static final String SUB = "SUB"; + + public static final String DENY = "DENY"; + + public static final String PUB_SUB = "PUB|SUB"; + + public static final String SUB_PUB = "SUB|PUB"; + public static final int ACCESS_KEY_MIN_LENGTH = 6; public static final int SECRET_KEY_MIN_LENGTH = 6; diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java b/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java index 8ceb135fbf1f3f5fc07a83f68a981c22780d0506..dadcaa304aac45a460feb3e9a15654ffa46d6e8b 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java @@ -60,14 +60,14 @@ public class Permission { return Permission.DENY; } switch (permString.trim()) { - case "PUB": + case AclConstants.PUB: return Permission.PUB; - case "SUB": + case AclConstants.SUB: return Permission.SUB; - case "PUB|SUB": - case "SUB|PUB": + case AclConstants.PUB_SUB: + case AclConstants.SUB_PUB: return Permission.PUB | Permission.SUB; - case "DENY": + case AclConstants.DENY: return Permission.DENY; default: return Permission.DENY; @@ -89,6 +89,25 @@ public class Permission { } } + public static void checkResourcePerms(List resources) { + if (resources == null || resources.isEmpty()) { + return; + } + + for (String resource : resources) { + String[] items = StringUtils.split(resource, "="); + if (items.length != 2) { + throw new AclException(String.format("Parse Resource format error for %s.\n" + + "The expected resource format is 'Res=Perm'. For example: topicA=SUB", resource)); + } + + if (!AclConstants.DENY.equals(items[1].trim()) && Permission.DENY == Permission.parsePermFromString(items[1].trim())) { + throw new AclException(String.format("Parse resource permission error for %s.\n" + + "The expected permissions are 'SUB' or 'PUB' or 'SUB|PUB' or 'PUB|SUB'.", resource)); + } + } + } + public static boolean needAdminPerm(Integer code) { return ADMIN_CODE.contains(code); } diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java index c182d7eb6a44fa7364542102471479b0fb25d86c..078d34b10b02a84837ad6bb0b34dd884833cab9c 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java @@ -50,9 +50,9 @@ public class PlainPermissionManager { private String fileName = System.getProperty("rocketmq.acl.plain.file", DEFAULT_PLAIN_ACL_FILE); - private Map plainAccessResourceMap = new HashMap<>(); + private Map plainAccessResourceMap = new HashMap<>(); - private List globalWhiteRemoteAddressStrategy = new ArrayList<>(); + private List globalWhiteRemoteAddressStrategy = new ArrayList<>(); private RemoteAddressStrategyFactory remoteAddressStrategyFactory = new RemoteAddressStrategyFactory(); @@ -80,7 +80,7 @@ public class PlainPermissionManager { if (globalWhiteRemoteAddressesList != null && !globalWhiteRemoteAddressesList.isEmpty()) { for (int i = 0; i < globalWhiteRemoteAddressesList.size(); i++) { globalWhiteRemoteAddressStrategy.add(remoteAddressStrategyFactory. - getRemoteAddressStrategy(globalWhiteRemoteAddressesList.getString(i))); + getRemoteAddressStrategy(globalWhiteRemoteAddressesList.getString(i))); } } @@ -89,7 +89,7 @@ public class PlainPermissionManager { List plainAccessConfigList = accounts.toJavaList(PlainAccessConfig.class); for (PlainAccessConfig plainAccessConfig : plainAccessConfigList) { PlainAccessResource plainAccessResource = buildPlainAccessResource(plainAccessConfig); - plainAccessResourceMap.put(plainAccessResource.getAccessKey(),plainAccessResource); + plainAccessResourceMap.put(plainAccessResource.getAccessKey(), plainAccessResource); } } @@ -128,12 +128,17 @@ public class PlainPermissionManager { if (plainAccessConfig == null) { log.error("Parameter value plainAccessConfig is null,Please check your parameter"); - return false; + throw new AclException("Parameter value plainAccessConfig is null, Please check your parameter"); } + Permission.checkResourcePerms(plainAccessConfig.getTopicPerms()); + Permission.checkResourcePerms(plainAccessConfig.getGroupPerms()); + Map aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, Map.class); - + if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) { + throw new AclException(String.format("the %s file is not found or empty", fileHome + File.separator + fileName)); + } List> accounts = (List>) aclAccessConfigMap.get(AclConstants.CONFIG_ACCOUNTS); Map updateAccountMap = null; if (accounts != null) { @@ -164,8 +169,9 @@ public class PlainPermissionManager { return false; } - private Map createAclAccessConfigMap(Map existedAccountMap, PlainAccessConfig plainAccessConfig) { - + private Map createAclAccessConfigMap(Map existedAccountMap, + PlainAccessConfig plainAccessConfig) { + Map newAccountsMap = null; if (existedAccountMap == null) { newAccountsMap = new LinkedHashMap(); @@ -176,8 +182,8 @@ public class PlainPermissionManager { if (StringUtils.isEmpty(plainAccessConfig.getAccessKey()) || plainAccessConfig.getAccessKey().length() <= AclConstants.ACCESS_KEY_MIN_LENGTH) { throw new AclException(String.format( - "The accessKey=%s cannot be null and length should longer than 6", - plainAccessConfig.getAccessKey())); + "The accessKey=%s cannot be null and length should longer than 6", + plainAccessConfig.getAccessKey())); } newAccountsMap.put(AclConstants.CONFIG_ACCESS_KEY, plainAccessConfig.getAccessKey()); @@ -218,8 +224,10 @@ public class PlainPermissionManager { } Map aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, - Map.class); - + Map.class); + if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) { + throw new AclException(String.format("the %s file is not found or empty", fileHome + File.separator + fileName)); + } List> accounts = (List>) aclAccessConfigMap.get("accounts"); if (accounts != null) { Iterator> itemIterator = accounts.iterator(); @@ -251,7 +259,9 @@ public class PlainPermissionManager { Map aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, Map.class); - + if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) { + throw new AclException(String.format("the %s file is not found or empty", fileHome + File.separator + fileName)); + } List globalWhiteRemoteAddrList = (List) aclAccessConfigMap.get(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS); if (globalWhiteRemoteAddrList != null) { @@ -259,7 +269,7 @@ public class PlainPermissionManager { globalWhiteRemoteAddrList.addAll(globalWhiteAddrsList); // Update globalWhiteRemoteAddr element in memeory map firstly - aclAccessConfigMap.put(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS,globalWhiteRemoteAddrList); + aclAccessConfigMap.put(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS, globalWhiteRemoteAddrList); if (AclUtils.writeDataObject(fileHome + File.separator + fileName, updateAclConfigFileVersion(aclAccessConfigMap))) { return true; } @@ -275,7 +285,7 @@ public class PlainPermissionManager { List configs = new ArrayList<>(); List whiteAddrs = new ArrayList<>(); JSONObject plainAclConfData = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, - JSONObject.class); + JSONObject.class); if (plainAclConfData == null || plainAclConfData.isEmpty()) { throw new AclException(String.format("%s file is not data", fileHome + File.separator + fileName)); } @@ -359,7 +369,7 @@ public class PlainPermissionManager { || plainAccessConfig.getSecretKey().length() <= AclConstants.SECRET_KEY_MIN_LENGTH) { throw new AclException(String.format( "The accessKey=%s and secretKey=%s cannot be null and length should longer than 6", - plainAccessConfig.getAccessKey(), plainAccessConfig.getSecretKey())); + plainAccessConfig.getAccessKey(), plainAccessConfig.getSecretKey())); } PlainAccessResource plainAccessResource = new PlainAccessResource(); plainAccessResource.setAccessKey(plainAccessConfig.getAccessKey()); @@ -375,7 +385,7 @@ public class PlainPermissionManager { Permission.parseResourcePerms(plainAccessResource, true, plainAccessConfig.getTopicPerms()); plainAccessResource.setRemoteAddressStrategy(remoteAddressStrategyFactory. - getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress())); + getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress())); return plainAccessResource; } diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java index 0c3548eeb55d6e6f6520c62b94788c1d9fdeba1f..6ec90ee3fad619f36517f1f96b5c40cf020d324b 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java @@ -193,7 +193,7 @@ public class RemoteAddressStrategyFactory { throw new AclException(String.format("RangeRemoteAddressStrategy netaddress examine scope Exception start is %s , end is %s", start, end)); } } - return this.end > 0 ? true : false; + return this.end > 0; } private boolean ipv6Analysis(String[] strArray, int index) { diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java index 253b5b241e05a9e900f5c730e0cec8094fbe2d98..c824065f7ff80241df142321642d9419c938c436 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.acl.common; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -165,4 +166,27 @@ public class PermissionTest { aclException.setStatus("netaddress examine scope Exception netaddress"); Assert.assertEquals(aclException.getStatus(),"netaddress examine scope Exception netaddress"); } + + @Test + public void checkResourcePermsNormalTest() { + Permission.checkResourcePerms(null); + Permission.checkResourcePerms(new ArrayList<>()); + Permission.checkResourcePerms(Arrays.asList("topicA=PUB")); + Permission.checkResourcePerms(Arrays.asList("topicA=PUB", "topicB=SUB", "topicC=PUB|SUB")); + } + + @Test(expected = AclException.class) + public void checkResourcePermsExceptionTest1() { + Permission.checkResourcePerms(Arrays.asList("topicA")); + } + + @Test(expected = AclException.class) + public void checkResourcePermsExceptionTest2() { + Permission.checkResourcePerms(Arrays.asList("topicA=")); + } + + @Test(expected = AclException.class) + public void checkResourcePermsExceptionTest3() { + Permission.checkResourcePerms(Arrays.asList("topicA=DENY1")); + } } diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java index 645e5227631dac402dc11ca9a44fa2a2fdd8b380..c3d41919a7e824efa93d911a7bd4ca81390f787b 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java @@ -546,6 +546,26 @@ public class PlainAccessValidatorTest { Assert.assertEquals(plainAccessValidator.updateAccessConfig(plainAccessConfig), false); } + @Test(expected = AclException.class) + public void createAndUpdateAccessAclYamlConfigExceptionTest() { + System.setProperty("rocketmq.home.dir", "src/test/resources"); + System.setProperty("rocketmq.acl.plain.file", "/conf/plain_acl_update_create.yml"); + + PlainAccessConfig plainAccessConfig = new PlainAccessConfig(); + plainAccessConfig.setAccessKey("RocketMQ33"); + plainAccessConfig.setSecretKey("123456789111"); + List topicPerms = new ArrayList(); + topicPerms.add("topicB=PUB"); + plainAccessConfig.setTopicPerms(topicPerms); + List groupPerms = new ArrayList(); + groupPerms.add("groupC=DENY1"); + plainAccessConfig.setGroupPerms(groupPerms); + + PlainAccessValidator plainAccessValidator = new PlainAccessValidator(); + // Create element in the acl access yaml config file + plainAccessValidator.updateAccessConfig(plainAccessConfig); + } + @Test public void updateGlobalWhiteAddrsNormalTest() { System.setProperty("rocketmq.home.dir", "src/test/resources"); diff --git a/broker/pom.xml b/broker/pom.xml index 64ae7d995519e235b5cd7ee44ad96ce037febd2a..8453452261d4d7aef31b8f35c66cdadd58ff9a44 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java b/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java index f4c30d5d648141fa6b00a16ca16d9d1d0486f9fc..877ddd83b403d95baf216d20d70314351ba60798 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java @@ -90,7 +90,7 @@ public class AssignmentManager { log.error("ScheduledTask: failed to pull TopicRouteData from NameServer", e); } } - }, 13000, this.brokerController.getBrokerConfig().getLoadBalancePollNameServerInterval(), TimeUnit.MILLISECONDS); + }, 200, this.brokerController.getBrokerConfig().getLoadBalancePollNameServerInterval(), TimeUnit.MILLISECONDS); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java b/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java index a74b6d66ce8517c08c608744ce489575da1e12cc..a89bace193e6fb64c60796172998a835bde4333e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java @@ -17,9 +17,9 @@ package org.apache.rocketmq.broker.mqtrace; public interface SendMessageHook { - public String hookName(); + String hookName(); - public void sendMessageBefore(final SendMessageContext context); + void sendMessageBefore(final SendMessageContext context); - public void sendMessageAfter(final SendMessageContext context); + void sendMessageAfter(final SendMessageContext context); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index d8f53e4542734ea7c37cd8c9af918cc374ac7bc2..724cf54c8137c57d1dd8a5d434e20e582674d7a7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -218,6 +218,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement String originMsgId = MessageAccessor.getOriginMessageId(msgExt); MessageAccessor.setOriginMessageId(msgInner, UtilAll.isBlank(originMsgId) ? msgExt.getMsgId() : originMsgId); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + CompletableFuture putMessageResult = this.brokerController.getMessageStore().asyncPutMessage(msgInner); return putMessageResult.thenApply((r) -> { if (r != null) { diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java index 68d58ef4400ffb5028d8c1b667dcf8ddfc236d22..9ea1eeee31b9995f35890a8b06e8ff765690b8ef 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java @@ -124,7 +124,7 @@ public class BrokerOuterAPITest { boolean success = Iterables.any(booleanList, new Predicate() { public boolean apply(Boolean input) { - return input ? true : false; + return input; } }); diff --git a/client/pom.xml b/client/pom.xml index 164082c9b26ac7e6fc6384b275266e3e5820057b..95ef4617dd1b68b56cbebc6fab990ea783c8d602 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -47,6 +47,10 @@ org.apache.commons commons-lang3 + + commons-codec + commons-codec + io.opentracing opentracing-api diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java index c0b04caea182b8c0fcb92ed6b9a9fe27cb783274..891c17e3ba2c05a06816ada0502548ef32b4e9e2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java +++ b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java @@ -30,9 +30,8 @@ public class ThreadLocalIndex { this.threadLocalIndex.set(index); } - index = Math.abs(index + 1); - this.threadLocalIndex.set(index); - return index; + this.threadLocalIndex.set(++index); + return Math.abs(index); } @Override diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 393bda928e5da1464acb9292c1f2b73aa7017cd4..c54399aa5ed24346fc801e4d9b15050ee7a2b997 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -23,6 +23,10 @@ import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAverage import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; +import org.apache.rocketmq.client.trace.TraceDispatcher; +import org.apache.rocketmq.client.trace.hook.ConsumeMessageTraceHookImpl; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; @@ -30,10 +34,13 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.NamespaceUtil; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; public class DefaultLitePullConsumer extends ClientConfig implements LitePullConsumer { + private final InternalLogger log = ClientLogger.getLog(); + private final DefaultLitePullConsumerImpl defaultLitePullConsumerImpl; /** @@ -153,6 +160,21 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon */ private String consumeTimestamp = UtilAll.timeMillisToHumanString3(System.currentTimeMillis() - (1000 * 60 * 30)); + /** + * Interface of asynchronous transfer data + */ + private TraceDispatcher traceDispatcher = null; + + /** + * The flag for message trace + */ + private boolean enableMsgTrace = false; + + /** + * The name value of message trace topic.If you don't config,you can use the default trace topic name. + */ + private String customizedTraceTopic; + /** * Default constructor. */ @@ -202,13 +224,24 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon @Override public void start() throws MQClientException { + setTraceDispatcher(); setConsumerGroup(NamespaceUtil.wrapNamespace(this.getNamespace(), this.consumerGroup)); this.defaultLitePullConsumerImpl.start(); + if (null != traceDispatcher) { + try { + traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel()); + } catch (MQClientException e) { + log.warn("trace dispatcher start failed ", e); + } + } } @Override public void shutdown() { this.defaultLitePullConsumerImpl.shutdown(); + if (null != traceDispatcher) { + traceDispatcher.shutdown(); + } } @Override @@ -490,4 +523,36 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon public void setConsumeTimestamp(String consumeTimestamp) { this.consumeTimestamp = consumeTimestamp; } + + public TraceDispatcher getTraceDispatcher() { + return traceDispatcher; + } + + public void setCustomizedTraceTopic(String customizedTraceTopic) { + this.customizedTraceTopic = customizedTraceTopic; + } + + private void setTraceDispatcher() { + if (isEnableMsgTrace()) { + try { + this.traceDispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, customizedTraceTopic, null); + this.defaultLitePullConsumerImpl.registerConsumeMessageHook( + new ConsumeMessageTraceHookImpl(traceDispatcher)); + } catch (Throwable e) { + log.error("system mqtrace hook init failed ,maybe can't send msg trace data"); + } + } + } + + public String getCustomizedTraceTopic() { + return customizedTraceTopic; + } + + public boolean isEnableMsgTrace() { + return enableMsgTrace; + } + + public void setEnableMsgTrace(boolean enableMsgTrace) { + this.enableMsgTrace = enableMsgTrace; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index d32dd158e3a46381b087a06e88925a28fcf3db35..f32215ab83df48350bd7cadd61d43b92fda4dc60 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -281,7 +281,7 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume private TraceDispatcher traceDispatcher = null; // force to use client rebalance - private boolean clientRebalance = false; + private boolean clientRebalance = true; /** * Default constructor. diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index 2128ffd0dd3fd44bddee8699671007d6ba9450fc..8884e4adfef9efd5caef99845d9e1ba4cd7c4da1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -268,17 +268,23 @@ public class MQAdminImpl { messageId.getOffset(), timeoutMillis); } - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, - long end) throws MQClientException, - InterruptedException { + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return queryMessage(topic, key, maxNum, begin, end, false); } + public QueryResult queryMessageByUniqKey(String topic, String uniqKey, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + + return queryMessage(topic, uniqKey, maxNum, begin, end, true); + } + public MessageExt queryMessageByUniqKey(String topic, String uniqKey) throws InterruptedException, MQClientException { - QueryResult qr = this.queryMessage(topic, uniqKey, 32, - MessageClientIDSetter.getNearlyTimeFromID(uniqKey).getTime() - 1000, Long.MAX_VALUE, true); + QueryResult qr = queryMessageByUniqKey(topic, uniqKey, 32, + MessageClientIDSetter.getNearlyTimeFromID(uniqKey).getTime() - 1000, Long.MAX_VALUE); if (qr != null && qr.getMessageList() != null && qr.getMessageList().size() > 0) { return qr.getMessageList().get(0); } else { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index c64c240f2f7b7c1af345565587335f27e3749218..2e73f1a5170278729b034f5972f1f6a568a93836 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -41,12 +41,15 @@ import org.apache.rocketmq.client.consumer.MessageQueueListener; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.TopicMessageQueueChangeListener; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore; import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.consumer.store.ReadOffsetType; import org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.ConsumeMessageContext; +import org.apache.rocketmq.client.hook.ConsumeMessageHook; import org.apache.rocketmq.client.hook.FilterMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; @@ -115,6 +118,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { */ private static final long PULL_TIME_DELAY_MILLS_WHEN_PAUSE = 1000; + private static final long PULL_TIME_DELAY_MILLS_ON_EXCEPTION = 3 * 1000; + private DefaultLitePullConsumer defaultLitePullConsumer; private final ConcurrentMap taskTable = @@ -142,6 +147,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private final MessageQueueLock messageQueueLock = new MessageQueueLock(); + private final ArrayList consumeMessageHookList = new ArrayList<>(); + // only for test purpose, will be modified by reflection in unit test. @SuppressWarnings("FieldMayBeFinal") private static boolean doNotUpdateTopicSubscribeInfoWhenSubscriptionChanged = false; @@ -161,6 +168,35 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { this.pullTimeDelayMillsWhenException = defaultLitePullConsumer.getPullTimeDelayMillsWhenException(); } + public void registerConsumeMessageHook(final ConsumeMessageHook hook) { + this.consumeMessageHookList.add(hook); + log.info("register consumeMessageHook Hook, {}", hook.hookName()); + } + + public void executeHookBefore(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageBefore(context); + } catch (Throwable e) { + log.error("consumeMessageHook {} executeHookBefore exception", hook.hookName(), e); + } + } + } + } + + public void executeHookAfter(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageAfter(context); + } catch (Throwable e) { + log.error("consumeMessageHook {} executeHookAfter exception", hook.hookName(), e); + } + } + } + } + private void checkServiceState() { if (this.serviceState != ServiceState.RUNNING) throw new IllegalStateException(NOT_RUNNING_EXCEPTION_MESSAGE); @@ -632,9 +668,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private long fetchConsumeOffset(MessageQueue messageQueue) { + private long fetchConsumeOffset(MessageQueue messageQueue) throws MQClientException { checkServiceState(); - long offset = this.rebalanceImpl.computePullFromWhere(messageQueue); + long offset = this.rebalanceImpl.computePullFromWhereWithException(messageQueue); return offset; } @@ -658,7 +694,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private long nextPullOffset(MessageQueue messageQueue) { + private long nextPullOffset(MessageQueue messageQueue) throws MQClientException { long offset = -1; long seekOffset = assignedMessageQueue.getSeekOffset(messageQueue); if (seekOffset != -1) { @@ -745,7 +781,15 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return; } - long offset = nextPullOffset(messageQueue); + long offset = 0L; + try { + offset = nextPullOffset(messageQueue); + } catch (MQClientException e) { + log.error("Failed to get next pull offset", e); + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_ON_EXCEPTION, TimeUnit.MILLISECONDS); + return; + } + if (this.isCancelled() || processQueue.isDropped()) { return; } @@ -759,7 +803,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { String topic = this.messageQueue.getTopic(); subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL); } - + PullResult pullResult = pull(messageQueue, subscriptionData, offset, defaultLitePullConsumer.getPullBatchSize()); if (this.isCancelled() || processQueue.isDropped()) { return; @@ -854,6 +898,18 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { null ); this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); + if (!this.consumeMessageHookList.isEmpty()) { + ConsumeMessageContext consumeMessageContext = new ConsumeMessageContext(); + consumeMessageContext.setNamespace(defaultLitePullConsumer.getNamespace()); + consumeMessageContext.setConsumerGroup(this.groupName()); + consumeMessageContext.setMq(mq); + consumeMessageContext.setMsgList(pullResult.getMsgFoundList()); + consumeMessageContext.setSuccess(false); + this.executeHookBefore(consumeMessageContext); + consumeMessageContext.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString()); + consumeMessageContext.setSuccess(true); + this.executeHookAfter(consumeMessageContext); + } return pullResult; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 64126b07ce267b6428a5779e34ae2308ec43619f..b478cb10932287da8ff8a8b001f5257979c51ca4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -293,8 +293,15 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { } } else { if (processQueue.isLocked()) { - if (!pullRequest.isLockedFirst()) { - final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue()); + if (!pullRequest.isPreviouslyLocked()) { + long offset = -1L; + try { + offset = this.rebalanceImpl.computePullFromWhereWithException(pullRequest.getMessageQueue()); + } catch (MQClientException e) { + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); + log.error("Failed to compute pull offset, pullResult: {}", pullRequest, e); + return; + } boolean brokerBusy = offset < pullRequest.getNextOffset(); log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}", pullRequest, offset, brokerBusy); @@ -303,7 +310,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { pullRequest, offset); } - pullRequest.setLockedFirst(true); + pullRequest.setPreviouslyLocked(true); pullRequest.setNextOffset(offset); } } else { @@ -577,8 +584,8 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { invisibleTime = 60000; } this.pullAPIWrapper.popAsync(popRequest.getMessageQueue(), invisibleTime, this.defaultMQPushConsumer.getPopBatchNums(), - popRequest.getConsumerGroup(), BROKER_SUSPEND_MAX_TIME_MILLIS, popCallback, true, popRequest.getInitMode(), - false, subscriptionData.getExpressionType(), subscriptionData.getSubString()); + popRequest.getConsumerGroup(), BROKER_SUSPEND_MAX_TIME_MILLIS, popCallback, true, popRequest.getInitMode(), + false, subscriptionData.getExpressionType(), subscriptionData.getSubString()); } catch (Exception e) { log.error("popAsync exception", e); this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException); @@ -590,7 +597,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { List msgFoundList = popResult.getMsgFoundList(); List msgListFilterAgain = msgFoundList; if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode() - && popResult.getMsgFoundList().size() > 0) { + && popResult.getMsgFoundList().size() > 0) { msgListFilterAgain = new ArrayList(popResult.getMsgFoundList().size()); for (MessageExt msg : popResult.getMsgFoundList()) { if (msg.getTags() != null) { @@ -773,12 +780,12 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { } void changePopInvisibleTimeAsync(String topic, String consumerGroup, String extraInfo, long invisibleTime, AckCallback callback) - throws MQClientException, RemotingException, InterruptedException, MQBrokerException { + throws MQClientException, RemotingException, InterruptedException, MQBrokerException { String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo); String brokerName = ExtraInfoUtil.getBrokerName(extraInfoStrs); int queueId = ExtraInfoUtil.getQueueId(extraInfoStrs); FindBrokerResult - findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); + findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); if (null == findBrokerResult) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); @@ -1094,19 +1101,19 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { // popInvisibleTime if (this.defaultMQPushConsumer.getPopInvisibleTime() < MIN_POP_INVISIBLE_TIME - || this.defaultMQPushConsumer.getPopInvisibleTime() > MAX_POP_INVISIBLE_TIME) { + || this.defaultMQPushConsumer.getPopInvisibleTime() > MAX_POP_INVISIBLE_TIME) { throw new MQClientException( - "popInvisibleTime Out of range [" + MIN_POP_INVISIBLE_TIME + ", " + MAX_POP_INVISIBLE_TIME + "]" - + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), - null); + "popInvisibleTime Out of range [" + MIN_POP_INVISIBLE_TIME + ", " + MAX_POP_INVISIBLE_TIME + "]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); } // popBatchNums if (this.defaultMQPushConsumer.getPopBatchNums() <= 0 || this.defaultMQPushConsumer.getPopBatchNums() > 32) { throw new MQClientException( - "popBatchNums Out of range [1, 32]" - + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), - null); + "popBatchNums Out of range [1, 32]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java index 71d1fdbe927604aa52dc87e5f2087454b343494b..b90192b99257f804801bd78cb46c2eef2e96ff10 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java @@ -24,14 +24,14 @@ public class PullRequest implements MessageRequest { private MessageQueue messageQueue; private ProcessQueue processQueue; private long nextOffset; - private boolean lockedFirst = false; + private boolean previouslyLocked = false; - public boolean isLockedFirst() { - return lockedFirst; + public boolean isPreviouslyLocked() { + return previouslyLocked; } - public void setLockedFirst(boolean lockedFirst) { - this.lockedFirst = lockedFirst; + public void setPreviouslyLocked(boolean previouslyLocked) { + this.previouslyLocked = previouslyLocked; } public String getConsumerGroup() { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java index 7e78c9b57f8edebe001f588352aafe7cab618dc4..7a457c152bd0eaa76c728eaab8fb40f21979b6b7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; @@ -391,17 +392,7 @@ public abstract class RebalanceImpl { } private boolean getRebalanceResultFromBroker(final String topic, final boolean isOrder) { - String strategyName; - switch (messageModel) { - case BROADCASTING: - strategyName = null; - break; - case CLUSTERING: - strategyName = this.allocateMessageQueueStrategy.getName(); - break; - default: - return true; - } + String strategyName = this.allocateMessageQueueStrategy.getName(); Set messageQueueAssignments; try { messageQueueAssignments = this.mQClientFactory.queryAssignment(topic, consumerGroup, @@ -422,9 +413,6 @@ public abstract class RebalanceImpl { } } Set mqAll = null; - if (messageModel == MessageModel.BROADCASTING) { - mqAll = mqSet; - } boolean changed = this.updateMessageQueueAssignment(topic, messageQueueAssignments, isOrder); if (changed) { log.info("broker rebalanced result changed. allocateMessageQueueStrategyName={}, group={}, topic={}, clientId={}, assignmentSet={}", @@ -705,7 +693,14 @@ public abstract class RebalanceImpl { this.removeDirtyOffset(mq); ProcessQueue pq = createProcessQueue(); pq.setLocked(true); - long nextOffset = this.computePullFromWhere(mq); + long nextOffset = -1L; + try { + nextOffset = this.computePullFromWhereWithException(mq); + } catch (MQClientException e) { + log.info("doRebalance, {}, compute offset failed, {}", consumerGroup, mq); + continue; + } + if (nextOffset >= 0) { ProcessQueue pre = this.processQueueTable.putIfAbsent(mq, pq); if (pre != null) { @@ -774,8 +769,17 @@ public abstract class RebalanceImpl { public abstract void removeDirtyOffset(final MessageQueue mq); + /** + * When the network is unstable, using this interface may return wrong offset. + * It is recommended to use computePullFromWhereWithException instead. + * @param mq + * @return offset + */ + @Deprecated public abstract long computePullFromWhere(final MessageQueue mq); + public abstract long computePullFromWhereWithException(final MessageQueue mq) throws MQClientException; + public abstract int getConsumeInitMode(); public abstract void dispatchPullRequest(final List pullRequestList, final long delay); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java index c8f5cf1b1fa6feefd86ef2635251b5aa0e2aee4a..4d347113c7cdda822308e0e42dc026268d0ddfb8 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java @@ -74,8 +74,20 @@ public class RebalanceLitePullImpl extends RebalanceImpl { this.litePullConsumerImpl.getOffsetStore().removeOffset(mq); } + @Deprecated @Override public long computePullFromWhere(MessageQueue mq) { + long result = -1L; + try { + result = computePullFromWhereWithException(mq); + } catch (MQClientException e) { + log.warn("Compute consume offset exception, mq={}", mq); + } + return result; + } + + @Override + public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException { ConsumeFromWhere consumeFromWhere = litePullConsumerImpl.getDefaultLitePullConsumer().getConsumeFromWhere(); long result = -1; switch (consumeFromWhere) { @@ -118,7 +130,8 @@ public class RebalanceLitePullImpl extends RebalanceImpl { try { result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } else { try { @@ -126,7 +139,8 @@ public class RebalanceLitePullImpl extends RebalanceImpl { UtilAll.YYYYMMDDHHMMSS).getTime(); result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } } else { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java index a68fb04eefdbe6fb5ba92fa59af54b92aeab842e..e1c67926a9918568858bb29a21cc658bd5ad9ff0 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Set; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; import org.apache.rocketmq.client.consumer.MessageQueueListener; +import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; @@ -68,11 +69,17 @@ public class RebalancePullImpl extends RebalanceImpl { this.defaultMQPullConsumerImpl.getOffsetStore().removeOffset(mq); } + @Deprecated @Override public long computePullFromWhere(MessageQueue mq) { return 0; } + @Override + public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException { + return 0; + } + @Override public int getConsumeInitMode() { throw new UnsupportedOperationException("no initMode for Pull"); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java index 88c7c1e4770f4c5066886ad85f5e785a379f585d..09d1521654830b73e019c0a910e1f7a361fdf492 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java @@ -115,7 +115,7 @@ public class RebalancePushImpl extends RebalanceImpl { @Override public boolean clientRebalance(String topic) { // POPTODO order pop consume not implement yet - return defaultMQPushConsumerImpl.getDefaultMQPushConsumer().isClientRebalance() || defaultMQPushConsumerImpl.isConsumeOrderly(); + return defaultMQPushConsumerImpl.getDefaultMQPushConsumer().isClientRebalance() || defaultMQPushConsumerImpl.isConsumeOrderly() || MessageModel.BROADCASTING.equals(messageModel); } public boolean removeUnnecessaryPopMessageQueue(final MessageQueue mq, final PopProcessQueue pq) { @@ -149,8 +149,20 @@ public class RebalancePushImpl extends RebalanceImpl { this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq); } + @Deprecated @Override public long computePullFromWhere(MessageQueue mq) { + long result = -1L; + try { + result = computePullFromWhereWithException(mq); + } catch (MQClientException e) { + log.warn("Compute consume offset exception, mq={}", mq); + } + return result; + } + + @Override + public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException { long result = -1; final ConsumeFromWhere consumeFromWhere = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumeFromWhere(); final OffsetStore offsetStore = this.defaultMQPushConsumerImpl.getOffsetStore(); @@ -171,7 +183,8 @@ public class RebalancePushImpl extends RebalanceImpl { try { result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } } else { @@ -199,7 +212,8 @@ public class RebalancePushImpl extends RebalanceImpl { try { result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } else { try { @@ -207,7 +221,8 @@ public class RebalancePushImpl extends RebalanceImpl { UtilAll.YYYYMMDDHHMMSS).getTime(); result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } } else { diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java index b2b06452f62e136d29e26f00baa5febac44db115..b65ba46a397ca24059dd2e0ccf16a9bb6d32469c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java @@ -85,7 +85,6 @@ public class TraceDataEncoder { bean.setMsgId(line[5]); bean.setRetryTimes(Integer.parseInt(line[6])); bean.setKeys(line[7]); - bean.setClientHost(line[8]); subBeforeContext.setTraceBeans(new ArrayList(1)); subBeforeContext.getTraceBeans().add(bean); resList.add(subBeforeContext); @@ -123,10 +122,9 @@ public class TraceDataEncoder { bean.setKeys(line[7]); bean.setStoreHost(line[8]); bean.setMsgType(MessageType.values()[Integer.parseInt(line[9])]); - bean.setClientHost(line[10]); - bean.setTransactionId(line[11]); - bean.setTransactionState(LocalTransactionState.valueOf(line[12])); - bean.setFromTransactionCheck(Boolean.parseBoolean(line[13])); + bean.setTransactionId(line[10]); + bean.setTransactionState(LocalTransactionState.valueOf(line[11])); + bean.setFromTransactionCheck(Boolean.parseBoolean(line[12])); endTransactionContext.setTraceBeans(new ArrayList(1)); endTransactionContext.getTraceBeans().add(bean); @@ -166,8 +164,7 @@ public class TraceDataEncoder { .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getMsgType().ordinal()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getOffsetMsgId()).append(TraceConstants.CONTENT_SPLITOR)// - .append(ctx.isSuccess()).append(TraceConstants.CONTENT_SPLITOR)// - .append(bean.getClientHost()).append(TraceConstants.FIELD_SPLITOR); + .append(ctx.isSuccess()).append(TraceConstants.FIELD_SPLITOR);// } break; case SubBefore: { @@ -179,8 +176,7 @@ public class TraceDataEncoder { .append(ctx.getRequestId()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getRetryTimes()).append(TraceConstants.CONTENT_SPLITOR)// - .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)// - .append(bean.getClientHost()).append(TraceConstants.FIELD_SPLITOR);// + .append(bean.getKeys()).append(TraceConstants.FIELD_SPLITOR);// } } break; @@ -198,6 +194,7 @@ public class TraceDataEncoder { } } + break; case EndTransaction: { TraceBean bean = ctx.getTraceBeans().get(0); sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)// @@ -210,7 +207,6 @@ public class TraceDataEncoder { .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getStoreHost()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getMsgType().ordinal()).append(TraceConstants.CONTENT_SPLITOR)// - .append(bean.getClientHost()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getTransactionId()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getTransactionState().name()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.isFromTransactionCheck()).append(TraceConstants.FIELD_SPLITOR); diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java index 14fc360f11bb59b4cad696673d2ce1d1acf13863..e78d37ab2cdbd4c5f0ece7d43aca814f1f44eb20 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java @@ -17,10 +17,10 @@ package org.apache.rocketmq.client.trace; - - import java.util.ArrayList; import java.util.List; +import org.apache.commons.codec.Charsets; +import org.apache.rocketmq.common.message.MessageExt; public class TraceView { @@ -38,8 +38,9 @@ public class TraceView { private String groupName; private String status; - public static List decodeFromTraceTransData(String key, String messageBody) { + public static List decodeFromTraceTransData(String key, MessageExt messageExt) { List messageTraceViewList = new ArrayList(); + String messageBody = new String(messageExt.getBody(), Charsets.UTF_8); if (messageBody == null || messageBody.length() <= 0) { return messageTraceViewList; } @@ -56,8 +57,7 @@ public class TraceView { messageTraceView.setGroupName(context.getGroupName()); if (context.isSuccess()) { messageTraceView.setStatus("success"); - } - else { + } else { messageTraceView.setStatus("failed"); } messageTraceView.setKeys(traceBean.getKeys()); @@ -68,7 +68,7 @@ public class TraceView { messageTraceView.setOffSetMsgId(traceBean.getOffsetMsgId()); messageTraceView.setTimeStamp(context.getTimeStamp()); messageTraceView.setStoreHost(traceBean.getStoreHost()); - messageTraceView.setClientHost(traceBean.getClientHost()); + messageTraceView.setClientHost(messageExt.getBornHostString()); messageTraceViewList.add(messageTraceView); } return messageTraceViewList; diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java index 4f6f916ea972094b79786b7d095de6493bd35371..bce613987ff65753ffca83a99ef2cbc9fadc11a6 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java @@ -16,10 +16,10 @@ */ package org.apache.rocketmq.client.trace.hook; +import java.util.Map; import org.apache.rocketmq.client.consumer.listener.ConsumeReturnType; import org.apache.rocketmq.client.hook.ConsumeMessageContext; import org.apache.rocketmq.client.hook.ConsumeMessageHook; -import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; import org.apache.rocketmq.client.trace.TraceContext; import org.apache.rocketmq.client.trace.TraceDispatcher; import org.apache.rocketmq.client.trace.TraceBean; @@ -74,7 +74,6 @@ public class ConsumeMessageTraceHookImpl implements ConsumeMessageHook { traceBean.setStoreTime(msg.getStoreTimestamp());// traceBean.setBodyLength(msg.getStoreSize());// traceBean.setRetryTimes(msg.getReconsumeTimes());// - traceBean.setClientHost(((AsyncTraceDispatcher)localDispatcher).getHostConsumer().getmQClientFactory().getClientId()); traceContext.setRegionId(regionId);// beans.add(traceBean); } @@ -93,7 +92,7 @@ public class ConsumeMessageTraceHookImpl implements ConsumeMessageHook { TraceContext subBeforeContext = (TraceContext) context.getMqTraceContext(); if (subBeforeContext.getTraceBeans() == null || subBeforeContext.getTraceBeans().size() < 1) { - // If subbefore bean is null ,skip it + // If subBefore bean is null ,skip it return; } TraceContext subAfterContext = new TraceContext(); @@ -103,13 +102,16 @@ public class ConsumeMessageTraceHookImpl implements ConsumeMessageHook { subAfterContext.setRequestId(subBeforeContext.getRequestId());// subAfterContext.setSuccess(context.isSuccess());// - // Caculate the cost time for processing messages + // Calculate the cost time for processing messages int costTime = (int) ((System.currentTimeMillis() - subBeforeContext.getTimeStamp()) / context.getMsgList().size()); subAfterContext.setCostTime(costTime);// subAfterContext.setTraceBeans(subBeforeContext.getTraceBeans()); - String contextType = context.getProps().get(MixAll.CONSUME_CONTEXT_TYPE); - if (contextType != null) { - subAfterContext.setContextCode(ConsumeReturnType.valueOf(contextType).ordinal()); + Map props = context.getProps(); + if (props != null) { + String contextType = props.get(MixAll.CONSUME_CONTEXT_TYPE); + if (contextType != null) { + subAfterContext.setContextCode(ConsumeReturnType.valueOf(contextType).ordinal()); + } } localDispatcher.append(subAfterContext); } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java index 4feb276284ed04f327c81e53029c2589bfcefe7b..80c7babdaa61b8e24a576b6ac7220e5b5fc04d00 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java @@ -60,7 +60,6 @@ public class SendMessageTraceHookImpl implements SendMessageHook { traceBean.setStoreHost(context.getBrokerAddr()); traceBean.setBodyLength(context.getMessage().getBody().length); traceBean.setMsgType(context.getMsgType()); - traceBean.setClientHost(((AsyncTraceDispatcher)localDispatcher).getHostProducer().getmQClientFactory().getClientId()); tuxeContext.getTraceBeans().add(traceBean); } diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index 4190e286a29358172527668e2056dc2864f627d1..7c3c501e0549b9dbb8694d980b014fb3dd0bd88b 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -57,6 +57,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.junit.After; @@ -73,6 +74,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.nullable; @@ -80,7 +82,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@RunWith(MockitoJUnitRunner.Silent.class) public class DefaultMQPushConsumerTest { private String consumerGroup; private String topic = "FooBar"; @@ -105,6 +107,7 @@ public class DefaultMQPushConsumerTest { pushConsumer = new DefaultMQPushConsumer(consumerGroup); pushConsumer.setNamesrvAddr("127.0.0.1:9876"); pushConsumer.setPullInterval(60 * 1000); + pushConsumer.setClientRebalance(false); pushConsumer.registerMessageListener(new MessageListenerConcurrently() { @Override @@ -121,8 +124,10 @@ public class DefaultMQPushConsumerTest { mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true))); factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory); doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); + doReturn(null).when(mQClientFactory).queryAssignment(anyString(), anyString(), anyString(), any(MessageModel.class), anyInt()); rebalanceImpl = spy(pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl()); + doReturn(123L).when(rebalanceImpl).computePullFromWhereWithException(any(MessageQueue.class)); Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); field.setAccessible(true); field.set(pushConsumerImpl, rebalanceImpl); @@ -174,7 +179,6 @@ public class DefaultMQPushConsumerTest { Set messageQueueSet = new HashSet(); messageQueueSet.add(createPullRequest().getMessageQueue()); pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet); - doReturn(123L).when(rebalanceImpl).computePullFromWhere(any(MessageQueue.class)); } @After @@ -286,7 +290,7 @@ public class DefaultMQPushConsumerTest { pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, - ConsumeConcurrentlyContext context) { + ConsumeConcurrentlyContext context) { assertThat(msgs.get(0).getBody()).isEqualTo(msgBody); countDownLatch.countDown(); try { @@ -345,4 +349,21 @@ public class DefaultMQPushConsumerTest { } return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); } + + @Test + public void testPullMessage_ExceptionOccursWhenComputePullFromWhere() throws MQClientException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + final MessageExt[] messageExts = new MessageExt[1]; + pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService( + new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), + (msgs, context) -> { + messageExts[0] = msgs.get(0); + return null; + })); + + pushConsumer.getDefaultMQPushConsumerImpl().setConsumeOrderly(true); + PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); + pullMessageService.executePullRequestImmediately(createPullRequest()); + assertThat(messageExts[0]).isNull(); + } } diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java index b7a42e153397da78139ec6388f9cbe403e907c36..17b48eba51d939a0982b5181aeda9a6ef8e2cec0 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java @@ -32,6 +32,7 @@ import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -58,6 +59,11 @@ public class RebalancePushImplTest { private String topic = "TopicA"; private final String brokerName = "BrokerA"; + @Before + public void before() { + defaultMQPushConsumer.getDefaultMQPushConsumer().setClientRebalance(false); + } + @Test public void testMessageQueueChanged_CountThreshold() { RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING, diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java index c173b8ef790b1711e906fad8a2c8f3ff26e7b70f..ecf72ae44cfefdb682346d366b557b0a322f4077 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java @@ -21,7 +21,6 @@ import io.opentracing.mock.MockSpan; import io.opentracing.mock.MockTracer; import io.opentracing.tag.Tags; import java.io.ByteArrayOutputStream; -import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.util.Collections; import java.util.HashSet; @@ -75,6 +74,7 @@ import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; @@ -105,6 +105,27 @@ public class DefaultMQConsumerWithOpenTracingTest { factoryTable.forEach((s, instance) -> instance.shutdown()); factoryTable.clear(); + when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class), + anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) + .thenAnswer(new Answer() { + @Override + public PullResult answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + MessageClientExt messageClientExt = new MessageClientExt(); + messageClientExt.setTopic(topic); + messageClientExt.setQueueId(0); + messageClientExt.setMsgId("123"); + messageClientExt.setBody(new byte[]{'a'}); + messageClientExt.setOffsetMsgId("234"); + messageClientExt.setBornHost(new InetSocketAddress(8080)); + messageClientExt.setStoreHost(new InetSocketAddress(8080)); + PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); + ((PullCallback) mock.getArgument(4)).onSuccess(pullResult); + return pullResult; + } + }); + + consumerGroup = "FooBarGroup" + System.currentTimeMillis(); pushConsumer = new DefaultMQPushConsumer(consumerGroup); pushConsumer.getDefaultMQPushConsumerImpl().registerConsumeMessageHook( @@ -128,58 +149,20 @@ public class DefaultMQConsumerWithOpenTracingTest { // suppress updateTopicRouteInfoFromNameServer pushConsumer.changeInstanceNameToPID(); - mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true))); + mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true)); + FieldUtils.writeDeclaredField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true); + mQClientFactory = spy(mQClientFactory); factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory); doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); - rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl())); - Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); - field.setAccessible(true); - field.set(pushConsumerImpl, rebalancePushImpl); - pushConsumer.subscribe(topic, "*"); - - pushConsumer.start(); - - field = DefaultMQPushConsumerImpl.class.getDeclaredField("mQClientFactory"); - field.setAccessible(true); - field.set(pushConsumerImpl, mQClientFactory); - - field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); - field.setAccessible(true); - field.set(mQClientFactory, mQClientAPIImpl); - - pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false)); - field = DefaultMQPushConsumerImpl.class.getDeclaredField("pullAPIWrapper"); - field.setAccessible(true); - field.set(pushConsumerImpl, pullAPIWrapper); - - pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().setmQClientFactory(mQClientFactory); - mQClientFactory.registerConsumer(consumerGroup, pushConsumerImpl); - - when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class), - anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) - .thenAnswer(new Answer() { - @Override - public PullResult answer(InvocationOnMock mock) throws Throwable { - PullMessageRequestHeader requestHeader = mock.getArgument(1); - MessageClientExt messageClientExt = new MessageClientExt(); - messageClientExt.setTopic(topic); - messageClientExt.setQueueId(0); - messageClientExt.setMsgId("123"); - messageClientExt.setBody(new byte[]{'a'}); - messageClientExt.setOffsetMsgId("234"); - messageClientExt.setBornHost(new InetSocketAddress(8080)); - messageClientExt.setStoreHost(new InetSocketAddress(8080)); - PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); - ((PullCallback) mock.getArgument(4)).onSuccess(pullResult); - return pullResult; - } - }); - doReturn(new FindBrokerResult("127.0.0.1:10911", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean()); + Set messageQueueSet = new HashSet(); messageQueueSet.add(createPullRequest().getMessageQueue()); - pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet); + pushConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + + pushConsumer.subscribe(topic, "*"); + pushConsumer.start(); } @After @@ -209,7 +192,8 @@ public class DefaultMQConsumerWithOpenTracingTest { assertThat(msg.getTopic()).isEqualTo(topic); assertThat(msg.getBody()).isEqualTo(new byte[]{'a'}); - assertThat(tracer.finishedSpans().size()).isEqualTo(1); + // wait until consumeMessageAfter hook of tracer is done surely. + waitAtMost(1, TimeUnit.SECONDS).until(() -> tracer.finishedSpans().size() == 1); MockSpan span = tracer.finishedSpans().get(0); assertThat(span.tags().get(Tags.MESSAGE_BUS_DESTINATION.getKey())).isEqualTo(topic); assertThat(span.tags().get(Tags.SPAN_KIND.getKey())).isEqualTo(Tags.SPAN_KIND_CONSUMER); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..67ae194b880ce5a497b7d0aaac72566276fb1ec5 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +import java.util.concurrent.ConcurrentMap; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.consumer.store.ReadOffsetType; +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQAdminImpl; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; +import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.impl.consumer.RebalanceImpl; +import org.apache.rocketmq.client.impl.consumer.RebalanceService; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.message.MessageClientExt; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.RPCHook; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultMQLitePullConsumerWithTraceTest { + + @Spy + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); + + @Mock + private MQClientAPIImpl mQClientAPIImpl; + @Mock + private MQAdminImpl mQAdminImpl; + + private AsyncTraceDispatcher asyncTraceDispatcher; + private DefaultMQProducer traceProducer; + private RebalanceImpl rebalanceImpl; + private OffsetStore offsetStore; + private DefaultLitePullConsumerImpl litePullConsumerImpl; + private String consumerGroup = "LitePullConsumerGroup"; + private String topic = "LitePullConsumerTest"; + private String brokerName = "BrokerA"; + private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); + + private String customerTraceTopic = "rmq_trace_topic_12345"; + + @Before + public void init() throws Exception { + Field field = MQClientInstance.class.getDeclaredField("rebalanceService"); + field.setAccessible(true); + RebalanceService rebalanceService = (RebalanceService) field.get(mQClientFactory); + field = RebalanceService.class.getDeclaredField("waitInterval"); + field.setAccessible(true); + field.set(rebalanceService, 100); + } + + @Test + public void testSubscribe_PollMessageSuccess_WithDefaultTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = createLitePullConsumerWithDefaultTraceTopic(); + try { + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testSubscribe_PollMessageSuccess_WithCustomizedTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = createLitePullConsumerWithCustomizedTraceTopic(); + try { + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + + private DefaultLitePullConsumer createLitePullConsumerWithDefaultTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setEnableMsgTrace(true); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.subscribe(topic, "*"); + suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; + } + + private DefaultLitePullConsumer createLitePullConsumerWithCustomizedTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setEnableMsgTrace(true); + litePullConsumer.setCustomizedTraceTopic(customerTraceTopic); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.subscribe(topic, "*"); + suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; + } + + private void initDefaultLitePullConsumer(DefaultLitePullConsumer litePullConsumer) throws Exception { + asyncTraceDispatcher = (AsyncTraceDispatcher) litePullConsumer.getTraceDispatcher(); + traceProducer = asyncTraceDispatcher.getTraceProducer(); + Field field = DefaultLitePullConsumer.class.getDeclaredField("defaultLitePullConsumerImpl"); + field.setAccessible(true); + litePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(litePullConsumerImpl, mQClientFactory); + + PullAPIWrapper pullAPIWrapper = litePullConsumerImpl.getPullAPIWrapper(); + field = PullAPIWrapper.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(pullAPIWrapper, mQClientFactory); + + Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + fieldTrace.setAccessible(true); + fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + field = MQClientInstance.class.getDeclaredField("mQAdminImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQAdminImpl); + + field = DefaultLitePullConsumerImpl.class.getDeclaredField("rebalanceImpl"); + field.setAccessible(true); + rebalanceImpl = (RebalanceImpl) field.get(litePullConsumerImpl); + field = RebalanceImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(rebalanceImpl, mQClientFactory); + + offsetStore = spy(litePullConsumerImpl.getOffsetStore()); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("offsetStore"); + field.setAccessible(true); + field.set(litePullConsumerImpl, offsetStore); + + traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); + + when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), + anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) + .thenAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + MessageClientExt messageClientExt = new MessageClientExt(); + messageClientExt.setTopic(topic); + messageClientExt.setQueueId(0); + messageClientExt.setMsgId("123"); + messageClientExt.setBody(new byte[] {'a'}); + messageClientExt.setOffsetMsgId("234"); + messageClientExt.setBornHost(new InetSocketAddress(8080)); + messageClientExt.setStoreHost(new InetSocketAddress(8080)); + PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); + return pullResult; + } + }); + + when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(new FindBrokerResult("127.0.0.1:10911", false)); + + doReturn(Collections.singletonList(mQClientFactory.getClientId())).when(mQClientFactory).findConsumerIdList(anyString(), anyString()); + + doReturn(123L).when(offsetStore).readOffset(any(MessageQueue.class), any(ReadOffsetType.class)); + + } + + private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, + List messageExtList) throws Exception { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (MessageExt messageExt : messageExtList) { + outputStream.write(MessageDecoder.encode(messageExt, false)); + } + return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); + } + + private MessageQueue createMessageQueue() { + MessageQueue messageQueue = new MessageQueue(); + messageQueue.setBrokerName(brokerName); + messageQueue.setQueueId(0); + messageQueue.setTopic(topic); + return messageQueue; + } + + private TopicRouteData createTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("BrokerA"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("BrokerA"); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("123"); + sendResult.setOffsetMsgId("123"); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + return sendResult; + } + + private static void suppressUpdateTopicRouteInfoFromNameServer(DefaultLitePullConsumer litePullConsumer) throws IllegalAccessException { + DefaultLitePullConsumerImpl defaultLitePullConsumerImpl = (DefaultLitePullConsumerImpl) FieldUtils.readDeclaredField(litePullConsumer, "defaultLitePullConsumerImpl", true); + if (litePullConsumer.getMessageModel() == MessageModel.CLUSTERING) { + litePullConsumer.changeInstanceNameToPID(); + } + MQClientInstance mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(litePullConsumer, (RPCHook) FieldUtils.readDeclaredField(defaultLitePullConsumerImpl, "rpcHook", true))); + ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); + factoryTable.put(litePullConsumer.buildMQClientId(), mQClientFactory); + doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); + } + +} \ No newline at end of file diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java index bac12ea0ced4eb08b5ba1542a704d962d4291b09..af5a7053957929e62e3d934299a3a6558742b5bf 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java @@ -18,7 +18,6 @@ package org.apache.rocketmq.client.trace; import org.apache.rocketmq.client.producer.LocalTransactionState; -import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageType; import org.junit.Assert; import org.junit.Before; @@ -50,8 +49,7 @@ public class TraceDataEncoderTest { .append(245).append(TraceConstants.CONTENT_SPLITOR) .append(MessageType.Normal_Msg.ordinal()).append(TraceConstants.CONTENT_SPLITOR) .append("0A9A002600002A9F0000000000002329").append(TraceConstants.CONTENT_SPLITOR) - .append(true).append(TraceConstants.CONTENT_SPLITOR) - .append(UtilAll.ipToIPv4Str(UtilAll.getIP())).append(TraceConstants.FIELD_SPLITOR) + .append(true).append(TraceConstants.FIELD_SPLITOR) .toString(); } @@ -104,7 +102,6 @@ public class TraceDataEncoderTest { traceBean.setTags("Tags"); traceBean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000"); traceBean.setStoreHost("127.0.0.1:10911"); - traceBean.setClientHost("127.0.0.1@41700"); traceBean.setMsgType(MessageType.Trans_msg_Commit); traceBean.setTransactionId("transactionId"); traceBean.setTransactionState(LocalTransactionState.COMMIT_MESSAGE); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java index 51a1543ee5e1235dfc3f98fb87361cff0ad7c9df..b1fdbaf965ad1d6fe30e35ea9fdcc672a52b29e2 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java @@ -17,7 +17,8 @@ package org.apache.rocketmq.client.trace; -import org.apache.rocketmq.common.UtilAll; +import org.apache.commons.codec.Charsets; +import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageType; import org.junit.Assert; import org.junit.Test; @@ -29,29 +30,30 @@ public class TraceViewTest { @Test public void testDecodeFromTraceTransData() { String messageBody = new StringBuilder() - .append("Pub").append(TraceConstants.CONTENT_SPLITOR) - .append(System.currentTimeMillis()).append(TraceConstants.CONTENT_SPLITOR) - .append("DefaultRegion").append(TraceConstants.CONTENT_SPLITOR) - .append("PID-test").append(TraceConstants.CONTENT_SPLITOR) - .append("topic-test").append(TraceConstants.CONTENT_SPLITOR) - .append("AC1415116D1418B4AAC217FE1B4E0000").append(TraceConstants.CONTENT_SPLITOR) - .append("Tags").append(TraceConstants.CONTENT_SPLITOR) - .append("Keys").append(TraceConstants.CONTENT_SPLITOR) - .append("127.0.0.1:10911").append(TraceConstants.CONTENT_SPLITOR) - .append(26).append(TraceConstants.CONTENT_SPLITOR) - .append(245).append(TraceConstants.CONTENT_SPLITOR) - .append(MessageType.Normal_Msg.ordinal()).append(TraceConstants.CONTENT_SPLITOR) - .append("0A9A002600002A9F0000000000002329").append(TraceConstants.CONTENT_SPLITOR) - .append(true).append(TraceConstants.CONTENT_SPLITOR) - .append(UtilAll.ipToIPv4Str(UtilAll.getIP())).append(TraceConstants.FIELD_SPLITOR) - .toString(); + .append("Pub").append(TraceConstants.CONTENT_SPLITOR) + .append(System.currentTimeMillis()).append(TraceConstants.CONTENT_SPLITOR) + .append("DefaultRegion").append(TraceConstants.CONTENT_SPLITOR) + .append("PID-test").append(TraceConstants.CONTENT_SPLITOR) + .append("topic-test").append(TraceConstants.CONTENT_SPLITOR) + .append("AC1415116D1418B4AAC217FE1B4E0000").append(TraceConstants.CONTENT_SPLITOR) + .append("Tags").append(TraceConstants.CONTENT_SPLITOR) + .append("Keys").append(TraceConstants.CONTENT_SPLITOR) + .append("127.0.0.1:10911").append(TraceConstants.CONTENT_SPLITOR) + .append(26).append(TraceConstants.CONTENT_SPLITOR) + .append(245).append(TraceConstants.CONTENT_SPLITOR) + .append(MessageType.Normal_Msg.ordinal()).append(TraceConstants.CONTENT_SPLITOR) + .append("0A9A002600002A9F0000000000002329").append(TraceConstants.CONTENT_SPLITOR) + .append(true).append(TraceConstants.FIELD_SPLITOR) + .toString(); + MessageExt message = new MessageExt(); + message.setBody(messageBody.getBytes(Charsets.UTF_8)); String key = "AC1415116D1418B4AAC217FE1B4E0000"; - List traceViews = TraceView.decodeFromTraceTransData(key, messageBody); + List traceViews = TraceView.decodeFromTraceTransData(key, message); Assert.assertEquals(traceViews.size(), 1); Assert.assertEquals(traceViews.get(0).getMsgId(), key); key = "AD4233434334AAC217FEFFD0000"; - traceViews = TraceView.decodeFromTraceTransData(key, messageBody); + traceViews = TraceView.decodeFromTraceTransData(key, message); Assert.assertEquals(traceViews.size(), 0); } } diff --git a/common/pom.xml b/common/pom.xml index 068de26dcb0c81984d5e7226ba15130345ed59d0..ac1d086b4ccf486b848beaae1813b70f81077911 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index 0d4aff0b0f9b4900ed2a4f33fedbd27fe0a1d6ab..48cd1582cc2abfb05d68c22ace352d8615009b7d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.common; public class MQVersion { - public static final int CURRENT_VERSION = Version.V4_8_0.ordinal(); + public static final int CURRENT_VERSION = Version.V4_9_0.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java index 200dec24813b33769fd2a8e502de492e60302855..1fe9aa106e9ab8692da9e0fb34b9ef06e0b43f57 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java @@ -20,10 +20,10 @@ public class PermName { public static final int PERM_PRIORITY = 0x1 << 3; public static final int PERM_READ = 0x1 << 2; public static final int PERM_WRITE = 0x1 << 1; - public static final int PERM_INHERIT = 0x1 << 0; + public static final int PERM_INHERIT = 0x1; public static String perm2String(final int perm) { - final StringBuffer sb = new StringBuffer("---"); + final StringBuilder sb = new StringBuilder("---"); if (isReadable(perm)) { sb.replace(0, 1, "R"); } diff --git a/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java b/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java index e72fb826013606e0ec9f636f06f06d038a2cb098..f57df26d6fe4794da0f2da75d62dc559ce37e901 100644 --- a/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java +++ b/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java @@ -20,7 +20,7 @@ package org.apache.rocketmq.common.hook; import java.nio.ByteBuffer; public interface FilterCheckHook { - public String hookName(); + String hookName(); - public boolean isFilterMatched(final boolean isUnitMode, final ByteBuffer byteBuffer); + boolean isFilterMatched(final boolean isUnitMode, final ByteBuffer byteBuffer); } diff --git a/distribution/bin/dledger/fast-try.sh b/distribution/bin/dledger/fast-try.sh index ff8a96036d1a4b759fd72ae5979cf56cbe9f8534..acdde71b8df6ed63669bafb6de5ee48800a71c8f 100644 --- a/distribution/bin/dledger/fast-try.sh +++ b/distribution/bin/dledger/fast-try.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -20,25 +20,25 @@ CURRENT_DIR="$(cd "$(dirname "$0")"; pwd)" RMQ_DIR=$CURRENT_DIR/../.. cd $RMQ_DIR -function startNameserver() { +startNameserver() { export JAVA_OPT_EXT=" -Xms512m -Xmx512m " nohup bin/mqnamesrv & } -function startBroker() { +startBroker() { export JAVA_OPT_EXT=" -Xms1g -Xmx1g " conf_name=$1 nohup bin/mqbroker -c $conf_name & } -function stopNameserver() { +stopNameserver() { PIDS=$(ps -ef|grep java|grep NamesrvStartup|grep -v grep|awk '{print $2}') if [ ! -z "$PIDS" ]; then kill -s TERM $PIDS fi } -function stopBroker() { +stopBroker() { conf_name=$1 PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}') i=1 @@ -46,7 +46,7 @@ function stopBroker() { do echo "Waiting to kill ..." kill -s TERM $PIDS - ((i=$i+1)) + i=`expr $i + 1` sleep 2 PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}') done @@ -56,7 +56,7 @@ function stopBroker() { fi } -function stopAll() { +stopAll() { ps -ef|grep java|grep BrokerStartup|grep -v grep|awk '{print $2}'|xargs kill stopNameserver stopBroker ./conf/dledger/broker-n0.conf @@ -64,18 +64,18 @@ function stopAll() { stopBroker ./conf/dledger/broker-n2.conf } -function startAll() { +startAll() { startNameserver startBroker ./conf/dledger/broker-n0.conf startBroker ./conf/dledger/broker-n1.conf startBroker ./conf/dledger/broker-n2.conf } -function checkConf() { +checkConf() { if [ ! -f ./conf/dledger/broker-n0.conf -o ! -f ./conf/dledger/broker-n1.conf -o ! -f ./conf/dledger/broker-n2.conf ]; then echo "Make sure the ./conf/dledger/broker-n0.conf, ./conf/dledger/broker-n1.conf, ./conf/dledger/broker-n2.conf exists" - exit -1 - fi + exit 1 + fi } @@ -83,7 +83,7 @@ function checkConf() { ## Main if [ $# -lt 1 ]; then echo "Usage: sh $0 start|stop" - exit -1 + exit 1 fi action=$1 checkConf diff --git a/distribution/bin/runbroker.cmd b/distribution/bin/runbroker.cmd index eab7e30f9d4522501a645684ba89249f6c47b920..753a11e9a48eb592301ed599509233624e2c7ba8 100644 --- a/distribution/bin/runbroker.cmd +++ b/distribution/bin/runbroker.cmd @@ -36,7 +36,7 @@ set "JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow" set "JAVA_OPT=%JAVA_OPT% -XX:+AlwaysPreTouch" set "JAVA_OPT=%JAVA_OPT% -XX:MaxDirectMemorySize=15g" set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages -XX:-UseBiasedLocking" -set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib" +set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib;%JAVA_HOME%\jre\lib\ext" set "JAVA_OPT=%JAVA_OPT% -cp %CLASSPATH%" "%JAVA%" %JAVA_OPT% %* \ No newline at end of file diff --git a/distribution/bin/runserver.cmd b/distribution/bin/runserver.cmd index 48e32bf2d7cb81eaeccfa5040b168bbd1af3847c..76865f7a160fc6d720019dbad916438fcd993660 100644 --- a/distribution/bin/runserver.cmd +++ b/distribution/bin/runserver.cmd @@ -31,7 +31,7 @@ set "JAVA_OPT=%JAVA_OPT% -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollect set "JAVA_OPT=%JAVA_OPT% -verbose:gc -Xloggc:"%USERPROFILE%\rmq_srv_gc.log" -XX:+PrintGCDetails" set "JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow" set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages" -set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib" +set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib;%JAVA_HOME%\jre\lib\ext" set "JAVA_OPT=%JAVA_OPT% -cp "%CLASSPATH%"" "%JAVA%" %JAVA_OPT% %* \ No newline at end of file diff --git a/distribution/pom.xml b/distribution/pom.xml index d5ab667229101dc4696ffb767495844541d3f362..a5698bca1c614dfafce9590993a9923e265d346e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/docs/cn/RocketMQ_Example.md b/docs/cn/RocketMQ_Example.md index fd36fd1c312beb2b225f67ffd0b24a5b0c28fb3e..d924ce1d7d1f54e8c68ed63ddaa040ad5445f13d 100644 --- a/docs/cn/RocketMQ_Example.md +++ b/docs/cn/RocketMQ_Example.md @@ -446,7 +446,7 @@ public class ScheduledMessageConsumer { public ConsumeConcurrentlyStatus consumeMessage(List messages, ConsumeConcurrentlyContext context) { for (MessageExt message : messages) { // Print approximate delay time period - System.out.println("Receive message[msgId=" + message.getMsgId() + "] " + (System.currentTimeMillis() - message.getStoreTimestamp()) + "ms later"); + System.out.println("Receive message[msgId=" + message.getMsgId() + "] " + (System.currentTimeMillis() - message.getBornTimestamp()) + "ms later"); } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } diff --git a/docs/cn/design.md b/docs/cn/design.md index dcd39026f2648e85edd3ad2ae54e7d3fa6b76f72..1c32521ed5530766a2888a66779e30723bb9bf20 100644 --- a/docs/cn/design.md +++ b/docs/cn/design.md @@ -58,7 +58,7 @@ code |int | 请求操作码，应答方根据不同的请求码进行不同的 language | LanguageCode | 请求方实现的语言 | 应答方实现的语言 version | int | 请求方程序的版本 | 应答方程序的版本 opaque | int |相当于requestId，在同一个连接上的不同请求标识码，与响应消息中的相对应 | 应答不做修改直接返回 -flag | int | 区分是普通RPC还是onewayRPC得标志 | 区分是普通RPC还是onewayRPC得标志 +flag | int | 区分是普通RPC还是onewayRPC的标志 | 区分是普通RPC还是onewayRPC的标志 remark | String | 传输自定义文本信息 | 传输自定义文本信息 extFields | HashMap | 请求自定义扩展信息 | 响应自定义扩展信息 diff --git a/docs/cn/operation.md b/docs/cn/operation.md index d8c0bb151d4dee4936f52c8a64e74368601e64df..bc2d38527fce649921818ce19928cbe07c01462b 100644 --- a/docs/cn/operation.md +++ b/docs/cn/operation.md @@ -26,7 +26,7 @@ The Name Server boot success... $ nohup sh bin/mqbroker -n localhost:9876 & ### 验证Name Server 是否启动成功，例如Broker的IP为：192.168.1.2，且名称为broker-a -$ tail -f ~/logs/rocketmqlogs/Broker.log +$ tail -f ~/logs/rocketmqlogs/broker.log The broker[broker-a, 192.169.1.2:10911] boot success... ``` diff --git a/docs/en/Design_LoadBlancing.md b/docs/en/Design_LoadBlancing.md index e6bb66f8ac35710a4708063f8fdb55f0634054cb..971e8b5dd45b73544d3fbc8b389ba010ad052387 100644 --- a/docs/en/Design_LoadBlancing.md +++ b/docs/en/Design_LoadBlancing.md @@ -1,22 +1,24 @@ ## Load Balancing Load balancing in RocketMQ is accomplished on Client side. Specifically, it can be divided into load balancing at Producer side when sending messages and load balancing at Constumer side when subscribing messages. -### 1 Producer Load Balancing +### Producer Load Balancing When the Producer sends a message, it will first find the specified TopicPublishInfo according to Topic. After getting the routing information of TopicPublishInfo, the RocketMQ client will select a queue (MessageQueue) from the messageQueue List in TopicPublishInfo to send the message by default.Specific fault-tolerant strategies are defined in the MQFaultStrategy class. Here is a sendLatencyFaultEnable switch variable, which, if turned on, filters out the Broker agent of not available on the basis of randomly gradually increasing modular arithmetic selection. The so-called "latencyFault Tolerance" refers to a certain period of time to avoid previous failures. For example, if the latency of the last request exceeds 550 Lms, it will evade 3000 Lms; if it exceeds 1000L, it will evade 60000 L; if it is closed, it will choose a queue (MessageQueue) to send messages by randomly gradually increasing modular arithmetic, and the latencyFault Tolerance mechanism is the key to achieve high availability of message sending. -### 2 Consumer Load Balancing +### Consumer Load Balancing In RocketMQ, the two consumption modes (Push/Pull) on the Consumer side are both based on the pull mode to get the message, while in the Push mode it is only a kind of encapsulation of the pull mode, which is essentially implemented as the message pulling thread after pulling a batch of messages from the server. After submitting to the message consuming thread pool, it continues to try again to pull the message to the server. If the message is not pulled, the pull is delayed and continues. In both pull mode based consumption patterns (Push/Pull), the Consumer needs to know which message queue - queue from the Broker side to get the message. Therefore, it is necessary to do load balancing on the Consumer side, that is, which Consumer consumption is allocated to the same ConsumerGroup by more than one MessageQueue on the Broker side. - 1， Heartbeat Packet Sending on Consumer side +#### 1 Heartbeat Packet Sending on Consumer side After Consumer is started, it continuously sends heartbeat packets to all Broker instances in the RocketMQ cluster via timing task (which contains the message consumption group name, subscription relationship collection,Message communication mode and the value of the client id,etc). After receiving the heartbeat message from Consumer, Broker side maintains it in Consumer Manager's local caching variable—consumerTable, At the same time, the encapsulated client network channel information is stored in the local caching variable—channelInfoTable, which can provide metadata information for the later load balancing of Consumer. -2，Core Class for Load Balancing on Consumer side—RebalanceImpl +#### 2 Core Class for Load Balancing on Consumer side—RebalanceImpl Starting the MQClientInstance instance in the startup process of the Consumer instance will complete the start of the load balancing service thread-RebalanceService (executed every 20 s). By looking at the source code, we can find that the run () method of the RebalanceService thread calls the rebalanceByTopic () method of the RebalanceImpl class, which is the core of the Consumer end load balancing. Here, rebalanceByTopic () method will do different logical processing depending on whether the consumer communication type is "broadcast mode" or "cluster mode". Here we mainly look at the main processing flow in cluster mode: -(1) Get the message consumption queue set (mqSet) under the Topic from the local cache variable—topicSubscribeInfoTable of the rebalanceImpl instance. -(2) Call mQClientFactory. findConsumerIdList () method to send a RPC communication request to Broker side to obtain the consumer Id list under the consumer group based on the parameters of topic and consumer group (consumer table constructed by Broker side based on the heartbeat data reported by the front consumer side responds and returns, business request code: GET_CONSUMER_LIST_BY_GROUP); -(3) First, the message consumption queue and the consumer Id under Topic are sorted, then the message queue to be pulled is calculated by using the message queue allocation strategy algorithm (default: the average allocation algorithm of the message queue). The average allocation algorithm here is similar to the paging algorithm. It ranks all MessageQueues like records. It ranks all consumers like pages. It calculates the average size of each page and the range of each page record. Finally, it traverses the whole range and calculates the records that the current consumer should allocate to (MessageQueue here). +##### 1) Get the message consumption queue set (mqSet) under the Topic from the local cache variable—topicSubscribeInfoTable of the rebalanceImpl instance. +##### 2) Call mQClientFactory. findConsumerIdList () method to send a RPC communication request to Broker side to obtain the consumer Id list under the consumer group based on the parameters of topic and consumer group (consumer table constructed by Broker side based on the heartbeat data reported by the front consumer side responds and returns, business request code: GET_CONSUMER_LIST_BY_GROUP); +##### 3) First, the message consumption queue and the consumer Id under Topic are sorted, then the message queue to be pulled is calculated by using the message queue allocation strategy algorithm (default: the average allocation algorithm of the message queue). The average allocation algorithm here is similar to the paging algorithm. It ranks all MessageQueues like records. It ranks all consumers like pages. It calculates the average size of each page and the range of each page record. Finally, it traverses the whole range and calculates the records that the current consumer should allocate to (MessageQueue here). ![Image text](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_8.png) -(4) Then, the updateProcessQueueTableInRebalance () method is invoked, which first compares the allocated message queue set (mqSet) with processQueueTable for filtering. + + +##### 4) Then, the updateProcessQueueTableInRebalance () method is invoked, which first compares the allocated message queue set (mqSet) with processQueueTable for filtering. ![Image text](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_9.png) - The red part of the processQueueTable annotation in the figure above @@ -37,6 +39,6 @@ Starting the MQClientInstance instance in the startup process of the Consumer in removeUnnecessaryMessageQueue () method to try to remove Entry as above; -Finally, a ProcessQueue object is created for each MessageQueue in the filtered message queue set (mqSet) and stored in the processQueueTable queue of RebalanceImpl (where the computePullFromWhere (MessageQueue mq) method of the RebalanceImpl instance is invoked to obtain the next progress consumption value offset of the MessageQueue object, which is then populated into the attribute of pullRequest object to be created next time.), and create pull request object—pullRequest to add to pull list—pullRequestList, and finally execute dispatchPullRequest () method. PullRequest object of Pull message is put into the blocking queue pullRequestQueue of PullMessageService service thread in turn, and the request of Pull message is sent to Broker end after the service thread takes out. Among them, we can focus on the contrast, RebalancePushImpl and RebalancePullImpl two implementation classes dispatchPullRequest () method is different, the method in RebalancePullImpl class is empty, thus answering the last question in the previous article. +Finally, a ProcessQueue object is created for each MessageQueue in the filtered message queue set (mqSet) and stored in the processQueueTable queue of RebalanceImpl (where the computePullFromWhere (MessageQueue mq) method of the RebalanceImpl instance is invoked to obtain the next progress consumption value offset of the MessageQueue object, which is then populated into the attribute of pullRequest object to be created next time.), and create pull request object—pullRequest to add to pull list—pullRequestList, and finally execute dispatchPullRequest () method. PullRequest object of Pull message is put into the blocking queue pullRequestQueue of PullMessageService service thread in turn, and the request of Pull message is sent to Broker end after the service thread takes out. The core design idea of message consumption queue is that a message consumption queue can only be consumed by one consumer in the same consumer group at the same time, and a message consumer can consume multiple message queues at the same time. diff --git a/example/pom.xml b/example/pom.xml index ca5189378b1f2fd7e751391bed01665607e18a64..b392d9a10c1b1cd80d8f1d93808c8b4b044808d4 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java index b6c6ae410f61afc4c9dd249ec5b56a0e74068c10..8a6429b2aa20ea00d5e7e51eb8824163e042d141 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java @@ -54,7 +54,7 @@ public class Consumer { final String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : "BenchmarkTest"; final int threadCount = commandLine.hasOption('w') ? Integer.parseInt(commandLine.getOptionValue('w')) : 20; final String groupPrefix = commandLine.hasOption('g') ? commandLine.getOptionValue('g').trim() : "benchmark_consumer"; - final String isSuffixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : "true"; + final String isSuffixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : "false"; final String filterType = commandLine.hasOption('f') ? commandLine.getOptionValue('f').trim() : null; final String expression = commandLine.hasOption('e') ? commandLine.getOptionValue('e').trim() : null; final double failRate = commandLine.hasOption('r') ? Double.parseDouble(commandLine.getOptionValue('r').trim()) : 0.0; @@ -190,7 +190,7 @@ public class Consumer { opt.setRequired(false); options.addOption(opt); - opt = new Option("p", "group prefix enable", true, "Consumer group name, Default: false"); + opt = new Option("p", "group suffix enable", true, "Consumer group suffix enable, Default: false"); opt.setRequired(false); options.addOption(opt); diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java index d1fa5743ae52164cc096122009f5ee26a8fe5f0e..6975ab58b0d63532ceac8bc8534908dd44af7478 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java @@ -17,12 +17,14 @@ package org.apache.rocketmq.example.benchmark; import java.io.UnsupportedEncodingException; +import java.util.Arrays; import java.util.LinkedList; import java.util.Random; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.cli.CommandLine; @@ -58,9 +60,10 @@ public class Producer { final int tagCount = commandLine.hasOption('l') ? Integer.parseInt(commandLine.getOptionValue('l')) : 0; final boolean msgTraceEnable = commandLine.hasOption('m') && Boolean.parseBoolean(commandLine.getOptionValue('m')); final boolean aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a')); + final long messageNum = commandLine.hasOption('q') ? Long.parseLong(commandLine.getOptionValue('q')) : 0; - System.out.printf("topic: %s threadCount: %d messageSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s%n", - topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable); + System.out.printf("topic: %s threadCount: %d messageSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s messageQuantity: %d%n", + topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable, messageNum); final InternalLogger log = ClientLogger.getLog(); @@ -72,6 +75,16 @@ public class Producer { final LinkedList snapshotList = new LinkedList(); + final long[] msgNums = new long[threadCount]; + + if (messageNum > 0) { + Arrays.fill(msgNums, messageNum / threadCount); + long mod = messageNum % threadCount; + if (mod > 0) { + msgNums[0] += mod; + } + } + timer.scheduleAtFixedRate(new TimerTask() { @Override public void run() { @@ -85,14 +98,7 @@ public class Producer { timer.scheduleAtFixedRate(new TimerTask() { private void printStats() { if (snapshotList.size() >= 10) { - Long[] begin = snapshotList.getFirst(); - Long[] end = snapshotList.getLast(); - - final long sendTps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); - final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); - - System.out.printf("Current Time: %s Send TPS: %d Max RT(ms): %d Average RT(ms): %7.3f Send Failed: %d Response Failed: %d%n", - System.currentTimeMillis(), sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]); + doPrintStats(snapshotList, statsBenchmark, false); } } @@ -120,9 +126,14 @@ public class Producer { producer.start(); for (int i = 0; i < threadCount; i++) { + final long msgNumLimit = msgNums[i]; + if (messageNum > 0 && msgNumLimit == 0) { + break; + } sendThreadPool.execute(new Runnable() { @Override public void run() { + int num = 0; while (true) { try { final Message msg; @@ -198,10 +209,28 @@ public class Producer { } catch (InterruptedException ignored) { } } + if (messageNum > 0 && ++num >= msgNumLimit) { + break; + } } } }); } + try { + sendThreadPool.shutdown(); + sendThreadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + timer.cancel(); + if (snapshotList.size() > 1) { + doPrintStats(snapshotList, statsBenchmark, true); + } else { + System.out.printf("[Complete] Send Total: %d Send Failed: %d Response Failed: %d%n", + statsBenchmark.getSendRequestSuccessCount().get() + statsBenchmark.getSendRequestFailedCount().get(), + statsBenchmark.getSendRequestFailedCount().get(), statsBenchmark.getReceiveResponseFailedCount().get()); + } + producer.shutdown(); + } catch (InterruptedException e) { + log.error("[Exit] Thread Interrupted Exception", e); + } } public static Options buildCommandlineOptions(final Options options) { @@ -233,6 +262,10 @@ public class Producer { opt.setRequired(false); options.addOption(opt); + opt = new Option("q", "messageQuantity", true, "Send message quantity, Default: 0, running forever"); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -249,6 +282,23 @@ public class Producer { return msg; } + + private static void doPrintStats(final LinkedList snapshotList, final StatsBenchmarkProducer statsBenchmark, boolean done) { + Long[] begin = snapshotList.getFirst(); + Long[] end = snapshotList.getLast(); + + final long sendTps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); + final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); + + if (done) { + System.out.printf("[Complete] Send Total: %d Send TPS: %d Max RT(ms): %d Average RT(ms): %7.3f Send Failed: %d Response Failed: %d%n", + statsBenchmark.getSendRequestSuccessCount().get() + statsBenchmark.getSendRequestFailedCount().get(), + sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]); + } else { + System.out.printf("Current Time: %s Send TPS: %d Max RT(ms): %d Average RT(ms): %7.3f Send Failed: %d Response Failed: %d%n", + System.currentTimeMillis(), sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]); + } + } } class StatsBenchmarkProducer { diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/PopPushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/PopPushConsumer.java index 4b737f7010220f17aec63fd1ce01b6024e9e1fab..d7a2c70afb1f99cb09c742d0e7bd7e9edb7192d3 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/PopPushConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/PopPushConsumer.java @@ -55,6 +55,7 @@ public class PopPushConsumer { System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; }); + consumer.setClientRebalance(false); consumer.start(); System.out.printf("Consumer Started.%n"); } diff --git a/filter/pom.xml b/filter/pom.xml index c12dd1fb28d59df1d3d8b5d34617287a8fa3c42a..41bda9d0d22d2c50c10ccdf4b36263e8689d3083 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java index 53f7e1c297e3acea100512b068c1a852e4e09c26..c10250e3bbb896071b3ef08a9247b707930eba88 100644 --- a/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java +++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java @@ -36,8 +36,8 @@ public class SimpleCharStream { * Position in buffer. */ public int bufpos = -1; - protected int bufline[]; - protected int bufcolumn[]; + protected int[] bufline; + protected int[] bufcolumn; protected int column = 0; protected int line = 1; @@ -62,8 +62,8 @@ public class SimpleCharStream { protected void ExpandBuff(boolean wrapAround) { char[] newbuffer = new char[bufsize + 2048]; - int newbufline[] = new int[bufsize + 2048]; - int newbufcolumn[] = new int[bufsize + 2048]; + int[] newbufline = new int[bufsize + 2048]; + int[] newbufcolumn = new int[bufsize + 2048]; try { if (wrapAround) { diff --git a/logappender/pom.xml b/logappender/pom.xml index a6e979a83e0f73772b5f1cdfe5ea07ce1f2f1623..cac520b610360308adb90a4a0dbf67ea2ed12d92 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 rocketmq-logappender diff --git a/logging/pom.xml b/logging/pom.xml index 77d0f29a97c2a92947117ea5a7fce8ead5b190c7..8ebb24304529e907f918e3e77176b200693d6189 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java index 487682cd44b171e8d3fff8d41d790d1d2383cb50..0dc81d74e6749bcbcdcfc37b2e80e065faf10f2b 100755 --- a/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java @@ -116,7 +116,6 @@ public class Level implements Serializable { if (s.equals(OFF_NAME)) { return Level.OFF; } - if (s.equals(INFO_NAME)) { return Level.INFO; } diff --git a/namesrv/pom.xml b/namesrv/pom.xml index c5d6d05bc2ad03ef248c9e40ab61ca39b3ec2527..01ef5b60fe951dfccff97a80ebd0df8f14cf1b47 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java index 44a2b6f2610c57ade7000958633eb4c4ddd9d566..f8bc55e7aab38e1f27abc75c674c28eab0a9d02b 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -140,6 +140,11 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen final PutKVConfigRequestHeader requestHeader = (PutKVConfigRequestHeader) request.decodeCommandCustomHeader(PutKVConfigRequestHeader.class); + if (requestHeader.getNamespace() == null || requestHeader.getKey() == null) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("namespace or key is null"); + return response; + } this.namesrvController.getKvConfigManager().putKVConfig( requestHeader.getNamespace(), requestHeader.getKey(), diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 7f0e937996b31a6d830cec1a45ac9f71217895da..4688c04043633cf98658a7d7d55cd5dad6459d27 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 7c2a8f4ce92b1e39d057815209a99bd15fba94d0..c6e039e583c874465e192147d6420d9c013cef00 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 2012 org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -439,6 +439,12 @@ 3.10.0 test + + org.awaitility + awaitility + 4.1.0 + test + @@ -536,7 +542,7 @@ com.alibaba fastjson - 1.2.69 + 1.2.76 org.javassist diff --git a/remoting/pom.xml b/remoting/pom.xml index d9dab9efa651cb0c9474ee1f58a242aa9ef8db0f..765e1a93bc0f83a7d59a794619ba356f7cccda9c 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java index a16940e561c5cec23c9df021c429e1c64d500b88..d936c3bf696ab0399a323ea028e9ee12ee46ce5a 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java @@ -19,6 +19,7 @@ package org.apache.rocketmq.remoting.common; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; +import java.io.File; import java.io.IOException; import java.lang.reflect.Method; import java.net.Inet6Address; @@ -98,6 +99,10 @@ public class RemotingUtil { ArrayList ipv6Result = new ArrayList(); while (enumeration.hasMoreElements()) { final NetworkInterface networkInterface = enumeration.nextElement(); + if (isBridge(networkInterface)) { + continue; + } + final Enumeration en = networkInterface.getInetAddresses(); while (en.hasMoreElements()) { final InetAddress address = en.nextElement(); @@ -160,6 +165,19 @@ public class RemotingUtil { return sb.toString(); } + private static boolean isBridge(NetworkInterface networkInterface) { + try { + if (isLinuxPlatform()) { + String interfaceName = networkInterface.getName(); + File file = new File("/sys/class/net/" + interfaceName + "/bridge"); + return file.exists(); + } + } catch (SecurityException e) { + //Ignore + } + return false; + } + public static SocketChannel connect(SocketAddress remote) { return connect(remote, 1000 * 5); } diff --git a/srvutil/pom.xml b/srvutil/pom.xml index b95808b7c26004e8dcd85bff0c9a135aaecd4276..9c4544c0b25e55c1e817aba7c49f2574d0dd0dc6 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index ed54db1b24cb5ab35072a9ede25236605e11964e..23e9763d81402d43340a3db26bf29a654b7e9587 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index c13ad4cfa22be8a5829075523b2d4e96510ca558..dd520f4859c8ba971aacc73bb74eacd073be4a53 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -152,6 +152,8 @@ public class MessageStoreConfig { private boolean isEnableBatchPush = false; + private boolean enableScheduleMessageStats = true; + public boolean isDebugLockEnable() { return debugLockEnable; } @@ -722,4 +724,12 @@ public class MessageStoreConfig { public void setEnableBatchPush(boolean enableBatchPush) { isEnableBatchPush = enableBatchPush; } + + public boolean isEnableScheduleMessageStats() { + return enableScheduleMessageStats; + } + + public void setEnableScheduleMessageStats(boolean enableScheduleMessageStats) { + this.enableScheduleMessageStats = enableScheduleMessageStats; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 9a6e7a78a128fc605ea37ed69025686f452b23d5..9241ffe4238cece8af9f58a5be6e54c662ccd524 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -426,17 +426,18 @@ public class DLedgerCommitLog extends CommitLog { AppendFuture dledgerFuture; EncodeResult encodeResult; + encodeResult = this.messageSerializer.serialize(msg); + if (encodeResult.status != AppendMessageStatus.PUT_OK) { + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)); + } + putMessageLock.lock(); //spin or ReentrantLock ,depending on store config long elapsedTimeInLock; long queueOffset; try { beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); - encodeResult = this.messageSerializer.serialize(msg); queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); - encodeResult.setQueueOffsetKey(queueOffset); - if (encodeResult.status != AppendMessageStatus.PUT_OK) { - return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)); - } + encodeResult.setQueueOffsetKey(queueOffset, false); AppendEntryRequest request = new AppendEntryRequest(); request.setGroup(dLedgerConfig.getGroup()); request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); @@ -542,6 +543,12 @@ public class DLedgerCommitLog extends CommitLog { BatchAppendFuture dledgerFuture; EncodeResult encodeResult; + encodeResult = this.messageSerializer.serialize(messageExtBatch); + if (encodeResult.status != AppendMessageStatus.PUT_OK) { + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult + .status)); + } + putMessageLock.lock(); //spin or ReentrantLock ,depending on store config msgIdBuilder.setLength(0); long elapsedTimeInLock; @@ -549,12 +556,8 @@ public class DLedgerCommitLog extends CommitLog { long msgNum = 0; try { beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); - encodeResult = this.messageSerializer.serialize(messageExtBatch); - queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey); - if (encodeResult.status != AppendMessageStatus.PUT_OK) { - return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult - .status)); - } + queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); + encodeResult.setQueueOffsetKey(queueOffset, true); BatchAppendEntryRequest request = new BatchAppendEntryRequest(); request.setGroup(dLedgerConfig.getGroup()); request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); @@ -664,7 +667,7 @@ public class DLedgerCommitLog extends CommitLog { try { beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); - encodeResult.setQueueOffsetKey(queueOffset); + encodeResult.setQueueOffsetKey(queueOffset, false); AppendEntryRequest request = new AppendEntryRequest(); request.setGroup(dLedgerConfig.getGroup()); request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); @@ -779,7 +782,8 @@ public class DLedgerCommitLog extends CommitLog { long msgNum = 0; try { beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); - queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey); + queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); + encodeResult.setQueueOffsetKey(queueOffset, true); BatchAppendEntryRequest request = new BatchAppendEntryRequest(); request.setGroup(dLedgerConfig.getGroup()); request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); @@ -957,8 +961,15 @@ public class DLedgerCommitLog extends CommitLog { this.queueOffsetKey = queueOffsetKey; } - public void setQueueOffsetKey(long offset) { - data.putLong(MessageDecoder.QUEUE_OFFSET_POSITION, offset); + public void setQueueOffsetKey(long offset, boolean isBatch) { + if (!isBatch) { + this.data.putLong(MessageDecoder.QUEUE_OFFSET_POSITION, offset); + return; + } + + for (byte[] data : batchData) { + ByteBuffer.wrap(data).putLong(MessageDecoder.QUEUE_OFFSET_POSITION, offset++); + } } public byte[] getData() { @@ -977,8 +988,6 @@ public class DLedgerCommitLog extends CommitLog { // The maximum length of the message private final int maxMessageSize; - // Build Message Key - private final StringBuilder keyBuilder = new StringBuilder(); MessageSerializer(final int size) { this.maxMessageSize = size; @@ -1079,17 +1088,7 @@ public class DLedgerCommitLog extends CommitLog { } public EncodeResult serialize(final MessageExtBatch messageExtBatch) { - keyBuilder.setLength(0); - keyBuilder.append(messageExtBatch.getTopic()); - keyBuilder.append('-'); - keyBuilder.append(messageExtBatch.getQueueId()); - String key = keyBuilder.toString(); - - Long queueOffset = DLedgerCommitLog.this.topicQueueTable.get(key); - if (null == queueOffset) { - queueOffset = 0L; - DLedgerCommitLog.this.topicQueueTable.put(key, queueOffset); - } + String key = messageExtBatch.getTopic() + "-" + messageExtBatch.getQueueId(); int totalMsgLen = 0; ByteBuffer messagesByteBuff = messageExtBatch.wrap(); @@ -1154,7 +1153,7 @@ public class DLedgerCommitLog extends CommitLog { // 5 FLAG msgStoreItemMemory.putInt(flag); // 6 QUEUEOFFSET - msgStoreItemMemory.putLong(queueOffset++); + msgStoreItemMemory.putLong(0L); // 7 PHYSICALOFFSET msgStoreItemMemory.putLong(0); // 8 SYSFLAG @@ -1210,6 +1209,7 @@ public class DLedgerCommitLog extends CommitLog { this.sbr = sbr; } + @Override public synchronized void release() { super.release(); if (sbr != null) { diff --git a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java index ee994c37a6da3185309fef1e9e5dbeb8f3bf8adc..9057ebefdfe49b870f46b4cd593f87c9f885d085 100644 --- a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java +++ b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java @@ -115,6 +115,7 @@ public class ScheduleMessageService extends ConfigManager { public void start() { if (started.compareAndSet(false, true)) { + super.load(); this.timer = new Timer("ScheduleMessageTimerThread", true); for (Map.Entry entry : this.delayLevelTable.entrySet()) { Integer level = entry.getKey(); @@ -330,6 +331,12 @@ public class ScheduleMessageService extends ConfigManager { if (putMessageResult != null && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { + if (ScheduleMessageService.this.defaultMessageStore.getMessageStoreConfig().isEnableScheduleMessageStats()) { + ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); + ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), + putMessageResult.getAppendMessageResult().getWroteBytes()); + ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incBrokerPutNums(putMessageResult.getAppendMessageResult().getMsgNum()); + } continue; } else { // XXX: warn and notify me diff --git a/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java b/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java index fd860e6b9d7ea5d8a3d2c6e3a4fee714cda35d57..fa3c6bfcd8b9fd08035d5c7c409a10337f32b8d0 100644 --- a/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java @@ -40,6 +40,9 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; +import static org.apache.rocketmq.store.stats.BrokerStatsManager.BROKER_PUT_NUMS; +import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_NUMS; +import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_SIZE; import static org.assertj.core.api.Assertions.assertThat; @@ -112,6 +115,10 @@ public class ScheduleMessageServiceTest { @Test public void deliverDelayedMessageTimerTaskTest() throws Exception { + assertThat(messageStore.getMessageStoreConfig().isEnableScheduleMessageStats()).isTrue(); + + assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_NUMS, topic)).isNull(); + MessageExtBrokerInner msg = buildMessage(); int realQueueId = msg.getQueueId(); // set delayLevel,and send delay message @@ -141,6 +148,10 @@ public class ScheduleMessageServiceTest { // now,found the message assertThat(messageResult.getStatus()).isEqualTo(GetMessageStatus.FOUND); + // get the stats change + assertThat(messageStore.getBrokerStatsManager().getStatsItem(BROKER_PUT_NUMS, brokerConfig.getBrokerClusterName()).getValue().get()).isEqualTo(1); + assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_NUMS, topic).getValue().get()).isEqualTo(1L); + assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_SIZE, topic).getValue().get()).isEqualTo(messageResult.getBufferTotalSize()); // get the message body ByteBuffer byteBuffer = ByteBuffer.allocate(messageResult.getBufferTotalSize()); diff --git a/test/pom.xml b/test/pom.xml index 851d9ab9aa74888a555dc074353e6c0042b924db..41296501c28b649e170660c8490325641be836ce 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java new file mode 100644 index 0000000000000000000000000000000000000000..036f60e1db1285f9783c6ec6b50a5e24ccdfa51f --- /dev/null +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.test.client.rmq; + +import org.apache.rocketmq.test.listener.AbstractListener; + +public class RMQPopConsumer extends RMQNormalConsumer { + public RMQPopConsumer(String nsAddr, String topic, String subExpression, + String consumerGroup, AbstractListener listner) { + super(nsAddr, topic, subExpression, consumerGroup, listner); + } + + @Override + public void create() { + super.create(); + consumer.setClientRebalance(false); + } +} diff --git a/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java index 48508462668e8f523d5e5ab6e389c93d9a3e49ef..d530db98b0fff59679ea277b251335162f1d5366 100644 --- a/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java +++ b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java @@ -22,6 +22,7 @@ import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.test.client.rmq.RMQPopConsumer; import org.apache.rocketmq.test.client.rmq.RMQSqlConsumer; import org.apache.rocketmq.test.listener.AbstractListener; @@ -62,6 +63,15 @@ public class ConsumerFactory { consumer.start(); return consumer; } + public static RMQPopConsumer getRMQPopConsumer(String nsAddr, String consumerGroup, + String topic, String subExpression, + AbstractListener listener) { + RMQPopConsumer consumer = new RMQPopConsumer(nsAddr, topic, subExpression, + consumerGroup, listener); + consumer.create(); + consumer.start(); + return consumer; + } public static DefaultMQPullConsumer getRMQPullConsumer(String nsAddr, String consumerGroup) throws Exception { DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer(consumerGroup); diff --git a/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java index 44db782b5c53b82ece5db8fba626071f6f8407fb..e08967eb91de53826980b94f530c3e2ff299ae6c 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java @@ -33,7 +33,7 @@ public class FileUtil { this.fileName = fileName; } - public static void main(String args[]) { + public static void main(String[] args) { String filePath = FileUtil.class.getResource("/").getPath(); String fileName = "test.txt"; FileUtil fileUtil = new FileUtil(filePath, fileName); diff --git a/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java b/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java index 6edeecadb2a2d2ec19b55a14801f49bf0321409b..0c24427d9248788a4db636a14d114d6ff1f66196 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java @@ -76,7 +76,7 @@ public class MQWait { } } - public static void main(String args[]) { + public static void main(String[] args) { long start = System.currentTimeMillis(); MQWait.setCondition(new Condition() { diff --git a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java index 1c2bdac305781e3923bdb3bc334f03548f26639c..41cacb684c3c175fa8316a3e2ce10dc2dd546389 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java @@ -100,14 +100,14 @@ public final class RandomUtil { return n + res % (m - n); } - private static char getChar(int arg[]) { + private static char getChar(int[] arg) { int size = arg.length; int c = rd.nextInt(size / 2); c = c * 2; return (char) (getIntegerBetween(arg[c], arg[c + 1])); } - private static String getString(int n, int arg[]) { + private static String getString(int n, int[] arg) { StringBuilder res = new StringBuilder(); for (int i = 0; i < n; i++) { res.append(getChar(arg)); @@ -116,17 +116,17 @@ public final class RandomUtil { } public static String getStringWithCharacter(int n) { - int arg[] = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; + int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; return getString(n, arg); } public static String getStringWithNumber(int n) { - int arg[] = new int[] {'0', '9' + 1}; + int[] arg = new int[] {'0', '9' + 1}; return getString(n, arg); } public static String getStringWithNumAndCha(int n) { - int arg[] = new int[] {'a', 'z' + 1, 'A', 'Z' + 1, '0', '9' + 1}; + int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1, '0', '9' + 1}; return getString(n, arg); } diff --git a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java index 9eca28bbe7d90c8e0bf5f9e15525ce943735c9b8..3f71176d6f01db16000a49cf6fc47e5c6e57ce9c 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java @@ -45,16 +45,16 @@ public class RandomUtils { } public static String getStringWithNumber(int n) { - int arg[] = new int[] {'0', '9' + 1}; + int[] arg = new int[] {'0', '9' + 1}; return getString(n, arg); } public static String getStringWithCharacter(int n) { - int arg[] = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; + int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; return getString(n, arg); } - private static String getString(int n, int arg[]) { + private static String getString(int n, int[] arg) { StringBuilder res = new StringBuilder(); for (int i = 0; i < n; i++) { res.append(getChar(arg)); @@ -62,7 +62,7 @@ public class RandomUtils { return res.toString(); } - private static char getChar(int arg[]) { + private static char getChar(int[] arg) { int size = arg.length; int c = rd.nextInt(size / 2); c = c * 2; diff --git a/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java b/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java index 965d2ee6705c467d7cde0bdcd77c06a57c814fff..69bfd8f7d10ed5a07b24621cab801819f3805697 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java @@ -140,7 +140,7 @@ public class VerifyUtils { return rtExpect; } - public static void main(String args[]) { + public static void main(String[] args) { verifyBalance(400, 0.1f, 230, 190); } } diff --git a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java index 82ab461aa53607c475f1aff7f6adda1118be728f..bdd991a335fcd170da3c604dfa1101c5c326ee86 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java @@ -43,7 +43,7 @@ public class ListDataCollectorImpl implements DataCollector { return datas; } - public void resetData() { + public synchronized void resetData() { datas.clear(); unlockIncrement(); } @@ -67,7 +67,7 @@ public class ListDataCollectorImpl implements DataCollector { return Collections.frequency(datas, data) == 1; } - public Collection getAllDataWithoutDuplicate() { + public synchronized Collection getAllDataWithoutDuplicate() { return new HashSet(datas); } @@ -81,7 +81,7 @@ public class ListDataCollectorImpl implements DataCollector { return res; } - public void removeData(Object data) { + public synchronized void removeData(Object data) { datas.remove(data); } diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index a32da0320fb299b5dcb281147147901456075080..82420105e07079796e1b7e3a6a046fedcce74fd2 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -133,6 +133,7 @@ public class IntegrationTestBase { brokerConfig.setBrokerIP1("127.0.0.1"); brokerConfig.setNamesrvAddr(nsAddr); brokerConfig.setEnablePropertyFilter(true); + brokerConfig.setLoadBalancePollNameServerInterval(500); storeConfig.setStorePathRootDir(baseDir); storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog"); storeConfig.setMappedFileSizeCommitLog(COMMIT_LOG_SIZE); diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgStaticBalanceIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgStaticBalanceIT.java index 4c31f6a6cef7fb55c1db6b7a3a6520be1f56974e..7b2f09ed029a53c70a02b0c575f47b26cd88f9a0 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgStaticBalanceIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgStaticBalanceIT.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.test.client.consumer.balance; -import org.apache.log4j.Logger; import org.apache.rocketmq.test.base.BaseConf; import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; @@ -29,11 +28,13 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static com.google.common.truth.Truth.assertThat; public class NormalMsgStaticBalanceIT extends BaseConf { - private static Logger logger = Logger.getLogger(NormalMsgStaticBalanceIT.class); + private static Logger logger = LoggerFactory.getLogger(NormalMsgStaticBalanceIT.class); private RMQNormalProducer producer = null; private String topic = null; @@ -75,13 +76,12 @@ public class NormalMsgStaticBalanceIT extends BaseConf { @Test public void testFourConsumersBalance() { int msgSize = 600; - RMQNormalConsumer consumer1 = getConsumer(nsAddr, topic, "*", new RMQNormalListener()); - RMQNormalConsumer consumer2 = getConsumer(nsAddr, consumer1.getConsumerGroup(), topic, - "*", new RMQNormalListener()); - RMQNormalConsumer consumer3 = getConsumer(nsAddr, consumer1.getConsumerGroup(), topic, - "*", new RMQNormalListener()); - RMQNormalConsumer consumer4 = getConsumer(nsAddr, consumer1.getConsumerGroup(), topic, - "*", new RMQNormalListener()); + String consumerGroup = initConsumerGroup(); + logger.info("use group: {}", consumerGroup); + RMQNormalConsumer consumer1 = getConsumer(nsAddr, consumerGroup, topic, "*", new RMQNormalListener()); + RMQNormalConsumer consumer2 = getConsumer(nsAddr, consumerGroup, topic, "*", new RMQNormalListener()); + RMQNormalConsumer consumer3 = getConsumer(nsAddr, consumerGroup, topic, "*", new RMQNormalListener()); + RMQNormalConsumer consumer4 = getConsumer(nsAddr, consumerGroup, topic, "*", new RMQNormalListener()); TestUtils.waitForSeconds(waitTime); producer.send(msgSize); diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java index 74a9e04ce2c55a7e17a53a54f6cb1c2373d70079..df020c1121efbc012fe96b7545e58cd334938362 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java @@ -22,8 +22,9 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageRequestMode; import org.apache.rocketmq.logging.inner.Logger; import org.apache.rocketmq.test.base.BaseConf; -import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.test.client.rmq.RMQPopConsumer; +import org.apache.rocketmq.test.factory.ConsumerFactory; import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener; import org.apache.rocketmq.test.util.RandomUtil; import org.apache.rocketmq.test.util.VerifyUtils; @@ -64,12 +65,14 @@ public class PopSubCheckIT extends BaseConf { RMQNormalProducer producer = getProducer(nsAddr, topic); producer.getProducer().setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE); - RMQNormalConsumer consumer = getConsumer(nsAddr, group, topic, "*", new RMQNormalListener()); - for (String brokerAddr : new String[]{brokerController1.getBrokerAddr(), brokerController2.getBrokerAddr()}) { defaultMQAdminExt.setMessageRequestMode(brokerAddr, topic, group, MessageRequestMode.POP, 8, 60_000); } + RMQPopConsumer consumer = ConsumerFactory.getRMQPopConsumer(nsAddr, group, + topic, "*", new RMQNormalListener()); + mqClients.add(consumer); + int msgNum = 1; producer.send(msgNum); Assert.assertEquals("Not all sent succeeded", msgNum, producer.getAllUndupMsgBody().size()); @@ -80,7 +83,7 @@ public class PopSubCheckIT extends BaseConf { .containsExactlyElementsIn(producer.getAllMsgBody()); for (Object o : consumer.getListener().getAllOriginMsg()) { MessageClientExt msg = (MessageClientExt) o; - assertThat(msg.getProperty(MessageConst.PROPERTY_POP_CK)).isNotEmpty(); + assertThat(msg.getProperty(MessageConst.PROPERTY_POP_CK)).named("check pop meta").isNotEmpty(); } consumer.getListener().waitForMessageConsume(msgNum, 3_000 * 9); diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java index 564da5ceaf1cf701ea91639f6fb780de126bc76b..0edbdbe2e386c2bea5029ce571387ec794a22315 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java @@ -92,7 +92,7 @@ public class MulTagSubIT extends BaseConf { @Test public void testSubTwoTabAndMatchTwo() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; String subExpress = String.format("%s||%s", tags[0], tags[1]); int msgSize = 10; @@ -113,7 +113,7 @@ public class MulTagSubIT extends BaseConf { @Test public void testSubThreeTabAndMatchTwo() { - String tags[] = {"jueyin1", "jueyin2", "jueyin3"}; + String[] tags = {"jueyin1", "jueyin2", "jueyin3"}; String subExpress = String.format("%s||%s", tags[0], tags[1]); int msgSize = 10; @@ -135,7 +135,7 @@ public class MulTagSubIT extends BaseConf { @Test public void testNoMatch() { - String tags[] = {"jueyin1", "jueyin2", "jueyin3"}; + String[] tags = {"jueyin1", "jueyin2", "jueyin3"}; String subExpress = "no_match"; int msgSize = 10; diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java index 31321975b57b9c2c0e52096a9d0a6bd113b4ed0f..8de1b7d4e0110919151ae7d0d567aeb430fad5ff 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java @@ -84,7 +84,7 @@ public class TagMessageWithMulConsumerIT extends BaseConf { @Test public void testSendMessagesWithTwoTag() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; int msgSize = 10; TagMessage tagMessage = new TagMessage(tags, topic, msgSize); @@ -113,7 +113,7 @@ public class TagMessageWithMulConsumerIT extends BaseConf { @Test public void testTwoConsumerOneMatchOneOtherMatchAll() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; String sub1 = String.format("%s||%s", tags[0], tags[1]); String sub2 = String.format("%s|| noExist", tags[0]); int msgSize = 10; @@ -144,7 +144,7 @@ public class TagMessageWithMulConsumerIT extends BaseConf { @Test public void testSubKindsOf() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; String sub1 = String.format("%s||%s", tags[0], tags[1]); String sub2 = String.format("%s|| noExist", tags[0]); String sub3 = tags[0]; diff --git a/tools/pom.xml b/tools/pom.xml index eeccae81bf410768b49bb97d63ee6de4c9efdcad..abe8197de3974b6c3b1f52f9e5efbbf5078224dd 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index 6cdb960ccbca1b3a3fbb7beaeddc2434debc24c7..c27c85c66de2ae226038de951308e0251ed06f18 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -129,12 +129,18 @@ public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { } @Override - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, - long end) throws MQClientException, - InterruptedException { + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return defaultMQAdminExtImpl.queryMessage(topic, key, maxNum, begin, end); } + public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + + return defaultMQAdminExtImpl.queryMessageByUniqKey(topic, key, maxNum, begin, end); + } + @Override public void start() throws MQClientException { defaultMQAdminExtImpl.start(); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index 248f7abefb388069e1064a05edcfc2f7138d0549..8ae68cdecbedf40865f2c49f2c07b37689cd18b7 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -999,12 +999,18 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { } @Override - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, - long end) throws MQClientException, - InterruptedException { + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.mqClientInstance.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); } + public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, + long end) throws MQClientException, InterruptedException { + + return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(topic, key, maxNum, begin, end); + } + @Override public void updateConsumeOffset(String brokerAddr, String consumeGroup, MessageQueue mq, long offset) throws RemotingException, InterruptedException, MQBrokerException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java index 5038123561bf0aafb880ee0100d3e3323c2c49ae..872a130d9f52ccafd240b49c3b32850e2b56f505 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java @@ -38,7 +38,7 @@ import org.apache.rocketmq.tools.command.SubCommandException; public class CLusterSendMsgRTCommand implements SubCommand { - public static void main(String args[]) { + public static void main(String[] args) { } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java index 9ad075082f667823e779c7a5413a0bc9d0e2171e..8c7bec616d7068a8fab2c8c03d79cabcfb1e64cb 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.UtilAll; @@ -57,82 +58,43 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { } } - public static void queryById(final DefaultMQAdminExt admin, final String topic, - final String msgId) throws MQClientException, - RemotingException, MQBrokerException, InterruptedException, IOException { - MessageExt msg = admin.viewMessage(topic, msgId); - - String bodyTmpFilePath = createBodyFile(msg); - - System.out.printf("%-20s %s%n", - "Topic:", - msg.getTopic() - ); - - System.out.printf("%-20s %s%n", - "Tags:", - "[" + msg.getTags() + "]" - ); - - System.out.printf("%-20s %s%n", - "Keys:", - "[" + msg.getKeys() + "]" - ); - - System.out.printf("%-20s %d%n", - "Queue ID:", - msg.getQueueId() - ); - - System.out.printf("%-20s %d%n", - "Queue Offset:", - msg.getQueueOffset() - ); - - System.out.printf("%-20s %d%n", - "CommitLog Offset:", - msg.getCommitLogOffset() - ); - - System.out.printf("%-20s %d%n", - "Reconsume Times:", - msg.getReconsumeTimes() - ); - - System.out.printf("%-20s %s%n", - "Born Timestamp:", - UtilAll.timeMillisToHumanString2(msg.getBornTimestamp()) - ); - - System.out.printf("%-20s %s%n", - "Store Timestamp:", - UtilAll.timeMillisToHumanString2(msg.getStoreTimestamp()) - ); - - System.out.printf("%-20s %s%n", - "Born Host:", - RemotingHelper.parseSocketAddressAddr(msg.getBornHost()) - ); - - System.out.printf("%-20s %s%n", - "Store Host:", - RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()) - ); - - System.out.printf("%-20s %d%n", - "System Flag:", - msg.getSysFlag() - ); - - System.out.printf("%-20s %s%n", - "Properties:", - msg.getProperties() != null ? msg.getProperties().toString() : "" - ); - - System.out.printf("%-20s %s%n", - "Message Body Path:", - bodyTmpFilePath - ); + public static void queryById(final DefaultMQAdminExt admin, final String topic, final String msgId, + final boolean showAll) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException, IOException { + + QueryResult queryResult = admin.queryMessageByUniqKey(topic, msgId, 32, 0, Long.MAX_VALUE); + assert queryResult != null; + List list = queryResult.getMessageList(); + if (list == null || list.size() == 0) { + return; + } + list.sort((o1, o2) -> (int) (o1.getStoreTimestamp() - o2.getStoreTimestamp())); + for (int i = 0; i < (showAll ? list.size() : 1); i++) { + showMessage(admin, list.get(i), i); + } + } + + private static void showMessage(final DefaultMQAdminExt admin, MessageExt msg, int index) throws IOException { + String bodyTmpFilePath = createBodyFile(msg, index); + + final String strFormat = "%-20s %s%n"; + final String intFormat = "%-20s %d%n"; + + System.out.printf(strFormat, "Topic:", msg.getTopic()); + System.out.printf(strFormat, "Tags:", "[" + msg.getTags() + "]"); + System.out.printf(strFormat, "Keys:", "[" + msg.getKeys() + "]"); + System.out.printf(intFormat, "Queue ID:", msg.getQueueId()); + System.out.printf(intFormat, "Queue Offset:", msg.getQueueOffset()); + System.out.printf(intFormat, "CommitLog Offset:", msg.getCommitLogOffset()); + System.out.printf(intFormat, "Reconsume Times:", msg.getReconsumeTimes()); + System.out.printf(strFormat, "Born Timestamp:", UtilAll.timeMillisToHumanString2(msg.getBornTimestamp())); + System.out.printf(strFormat, "Store Timestamp:", UtilAll.timeMillisToHumanString2(msg.getStoreTimestamp())); + System.out.printf(strFormat, "Born Host:", RemotingHelper.parseSocketAddressAddr(msg.getBornHost())); + System.out.printf(strFormat, "Store Host:", RemotingHelper.parseSocketAddressAddr(msg.getStoreHost())); + System.out.printf(intFormat, "System Flag:", msg.getSysFlag()); + System.out.printf(strFormat, "Properties:", + msg.getProperties() != null ? msg.getProperties().toString() : ""); + System.out.printf(strFormat, "Message Body Path:", bodyTmpFilePath); try { List mtdList = admin.messageTrackDetail(msg); @@ -149,18 +111,21 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { } } - private static String createBodyFile(MessageExt msg) throws IOException { + private static String createBodyFile(MessageExt msg, int index) throws IOException { DataOutputStream dos = null; try { - String bodyTmpFilePath = "/tmp/rocketmq/msgbodys"; - File file = new File(bodyTmpFilePath); + StringBuffer bodyTmpFilePath = new StringBuffer("/tmp/rocketmq/msgbodys"); + File file = new File(bodyTmpFilePath.toString()); if (!file.exists()) { file.mkdirs(); } - bodyTmpFilePath = bodyTmpFilePath + "/" + msg.getMsgId(); - dos = new DataOutputStream(new FileOutputStream(bodyTmpFilePath)); + bodyTmpFilePath.append("/").append(msg.getMsgId()); + if (index > 0) { + bodyTmpFilePath.append("_" + index); + } + dos = new DataOutputStream(new FileOutputStream(bodyTmpFilePath.toString())); dos.write(msg.getBody()); - return bodyTmpFilePath; + return bodyTmpFilePath.toString(); } finally { if (dos != null) dos.close(); @@ -195,6 +160,10 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { opt.setRequired(true); options.addOption(opt); + opt = new Option("a", "showAll", false, "Print all message, the limit is 32"); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -202,11 +171,11 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { try { - defaultMQAdminExt = createMQAdminExt(rpcHook); final String msgId = commandLine.getOptionValue('i').trim(); final String topic = commandLine.getOptionValue('t').trim(); + final boolean showAll = commandLine.hasOption('a'); if (commandLine.hasOption('g') && commandLine.hasOption('d')) { final String consumerGroup = commandLine.getOptionValue('g').trim(); final String clientId = commandLine.getOptionValue('d').trim(); @@ -214,7 +183,7 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); System.out.printf("%s", result); } else { - queryById(defaultMQAdminExt, topic, msgId); + queryById(defaultMQAdminExt, topic, msgId, showAll); } } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java index bed2763dadcde8e06c9bababb8e9538f3b48e478..7382ff568dcf04668687acf37eceacbbce1f614f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java @@ -19,7 +19,6 @@ package org.apache.rocketmq.tools.command.message; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; -import org.apache.commons.codec.Charsets; import org.apache.commons.lang3.time.DateFormatUtils; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.exception.MQClientException; @@ -38,7 +37,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; - public class QueryMsgTraceByIdSubCommand implements SubCommand { @Override @@ -46,6 +44,10 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { Option opt = new Option("i", "msgId", true, "Message Id"); opt.setRequired(true); options.addOption(opt); + + opt = new Option("t", "traceTopic", true, "The name value of message trace topic"); + opt.setRequired(false); + options.addOption(opt); return options; } @@ -65,7 +67,11 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); try { final String msgId = commandLine.getOptionValue('i').trim(); - this.queryTraceByMsgId(defaultMQAdminExt, msgId); + String traceTopic = TopicValidator.RMQ_SYS_TRACE_TOPIC; + if (commandLine.hasOption('t')) { + traceTopic = commandLine.getOptionValue('t').trim(); + } + this.queryTraceByMsgId(defaultMQAdminExt, traceTopic, msgId); } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + "command failed", e); } finally { @@ -73,14 +79,14 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { } } - private void queryTraceByMsgId(final DefaultMQAdminExt admin, String msgId) - throws MQClientException, InterruptedException { + private void queryTraceByMsgId(final DefaultMQAdminExt admin, String traceTopic, String msgId) + throws MQClientException, InterruptedException { admin.start(); - QueryResult queryResult = admin.queryMessage(TopicValidator.RMQ_SYS_TRACE_TOPIC, msgId, 64, 0, System.currentTimeMillis()); + QueryResult queryResult = admin.queryMessage(traceTopic, msgId, 64, 0, System.currentTimeMillis()); List messageList = queryResult.getMessageList(); List traceViews = new ArrayList<>(); for (MessageExt message : messageList) { - List traceView = TraceView.decodeFromTraceTransData(msgId, new String(message.getBody(), Charsets.UTF_8)); + List traceView = TraceView.decodeFromTraceTransData(msgId, message); traceViews.addAll(traceView); } @@ -92,20 +98,20 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { for (TraceView traceView : traceViews) { if (traceView.getMsgType().equals(TraceType.Pub.name())) { System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", - "#Type", - "#ProducerGroup", - "#ClientHost", - "#SendTime", - "#CostTimes", - "#Status" + "#Type", + "#ProducerGroup", + "#ClientHost", + "#SendTime", + "#CostTimes", + "#Status" ); System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", - "Pub", - traceView.getGroupName(), - traceView.getClientHost(), - DateFormatUtils.format(traceView.getTimeStamp(), "yyyy-MM-dd HH:mm:ss"), - traceView.getCostTime() + "ms", - traceView.getStatus() + "Pub", + traceView.getGroupName(), + traceView.getClientHost(), + DateFormatUtils.format(traceView.getTimeStamp(), "yyyy-MM-dd HH:mm:ss"), + traceView.getCostTime() + "ms", + traceView.getStatus() ); System.out.printf("\n"); } @@ -124,22 +130,22 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { Iterator consumers = consumerTraceMap.keySet().iterator(); while (consumers.hasNext()) { System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", - "#Type", - "#ConsumerGroup", - "#ClientHost", - "#ConsumerTime", - "#CostTimes", - "#Status" + "#Type", + "#ConsumerGroup", + "#ClientHost", + "#ConsumerTime", + "#CostTimes", + "#Status" ); List consumerTraces = consumerTraceMap.get(consumers.next()); for (TraceView traceView : consumerTraces) { System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", - "Sub", - traceView.getGroupName(), - traceView.getClientHost(), - DateFormatUtils.format(traceView.getTimeStamp(), "yyyy-MM-dd HH:mm:ss"), - traceView.getCostTime() + "ms", - traceView.getStatus() + "Sub", + traceView.getGroupName(), + traceView.getClientHost(), + DateFormatUtils.format(traceView.getTimeStamp(), "yyyy-MM-dd HH:mm:ss"), + traceView.getCostTime() + "ms", + traceView.getStatus() ); } System.out.printf("\n"); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java index c770db01d427c9e33c17a8a6cc20a76ed6760dfc..a78a4a63ab0eb6495561eaf734f06cffd8a627c7 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java @@ -19,14 +19,23 @@ package org.apache.rocketmq.tools.command.topic; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + public class TopicRouteSubCommand implements SubCommand { + private static final String FORMAT = "%-45s %-32s %-50s %-10s %-11s %-5s%n"; + @Override public String commandName() { return "topicRoute"; @@ -43,6 +52,9 @@ public class TopicRouteSubCommand implements SubCommand { opt.setRequired(true); options.addOption(opt); + opt = new Option("l", "list", false, "Use list format to print data"); + opt.setRequired(false); + options.addOption(opt); return options; } @@ -58,12 +70,46 @@ public class TopicRouteSubCommand implements SubCommand { String topic = commandLine.getOptionValue('t').trim(); TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); - String json = topicRouteData.toJson(true); - System.out.printf("%s%n", json); + printData(topicRouteData, commandLine.hasOption('l')); } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); } finally { defaultMQAdminExt.shutdown(); } } -} + + private void printData(TopicRouteData topicRouteData, boolean useListFormat) { + if (!useListFormat) { + System.out.printf("%s%n", topicRouteData.toJson(true)); + return; + } + + int totalReadQueue = 0, totalWriteQueue = 0; + List queueDataList = topicRouteData.getQueueDatas(); + Map map = new HashMap<>(); + for (QueueData queueData : queueDataList) { + map.put(queueData.getBrokerName(), queueData); + } + queueDataList.sort(Comparator.comparing(QueueData::getBrokerName)); + + List brokerDataList = topicRouteData.getBrokerDatas(); + brokerDataList.sort(Comparator.comparing(BrokerData::getBrokerName)); + + System.out.printf(FORMAT, "#ClusterName", "#BrokerName", "#BrokerAddrs", "#ReadQueue", "#WriteQueue", "#Perm"); + + for (BrokerData brokerData : brokerDataList) { + String brokerName = brokerData.getBrokerName(); + QueueData queueData = map.get(brokerName); + totalReadQueue += queueData.getReadQueueNums(); + totalWriteQueue += queueData.getWriteQueueNums(); + System.out.printf(FORMAT, brokerData.getCluster(), brokerName, brokerData.getBrokerAddrs(), + queueData.getReadQueueNums(), queueData.getWriteQueueNums(), queueData.getPerm()); + } + + for (int i = 0; i < 158; i++) { + System.out.print("-"); + } + System.out.printf("%n"); + System.out.printf(FORMAT, "Total:", map.keySet().size(), "", totalReadQueue, totalWriteQueue, ""); + } +} \ No newline at end of file