提交 2b692c91 编写于 作者: D dongeforever

Merge branch 'develop'

Please do not create a Pull Request without creating an issue first.
## What is the purpose of the change ## What is the purpose of the change
XXXXX XXXXX
...@@ -12,11 +10,11 @@ XX ...@@ -12,11 +10,11 @@ XX
XXXX XXXX
Follow this checklist to help us incorporate your contribution quickly and easily. Follow this checklist to help us incorporate your contribution quickly and easily:
- [x] Make sure there is a Github issue filed for the change (usually before you start working on it). Trivial changes like typos do not require a Github issue. Your pull request should address just this issue, without pulling in other changes - one PR resolves one issue. - [x] Make sure there is a [JIRA issue](https://issues.apache.org/jira/projects/ROCKETMQ/issues/) filed for the change (usually before you start working on it). Trivial changes like typos do not require a JIRA issue. Your pull request should address just this issue, without pulling in other changes - one PR resolves one issue.
- [ ] Format the pull request title like `[ISSUE #123] Fix UnknownException when host config not exist`. Each commit in the pull request should have a meaningful subject line and body. - [ ] Format the pull request title like `[ROCKETMQ-XXX] Fix UnknownException when host config not exist`. Each commit in the pull request should have a meaningful subject line and body.
- [ ] Write a pull request description that is detailed enough to understand what the pull request does, how, and why. - [ ] Write a pull request description that is detailed enough to understand what the pull request does, how, and why.
- [ ] Write necessary unit-test(over 80% coverage) to verify your logic correction, more mock a little better when cross module dependency exist. If the new feature or significant change is committed, please remember to add integration-test in [test module](https://github.com/apache/rocketmq/tree/master/test). - [ ] Write necessary unit-test to verify your logic correction, more mock a little better when cross module dependency exist. If the new feature or significant change is committed, please remember to add integration-test in [test module](https://github.com/apache/rocketmq/tree/master/test).
- [ ] Run `mvn -B clean apache-rat:check findbugs:findbugs checkstyle:checkstyle` to make sure basic checks pass. Run `mvn clean install -DskipITs` to make sure unit-test pass. Run `mvn clean test-compile failsafe:integration-test` to make sure integration-test pass. - [ ] Run `mvn -B clean apache-rat:check findbugs:findbugs checkstyle:checkstyle` to make sure basic checks pass. Run `mvn clean install -DskipITs` to make sure unit-test pass. Run `mvn clean test-compile failsafe:integration-test` to make sure integration-test pass.
- [ ] If this contribution is large, please file an [Apache Individual Contributor License Agreement](http://www.apache.org/licenses/#clas). - [ ] If this contribution is large, please file an [Apache Individual Contributor License Agreement](http://www.apache.org/licenses/#clas).
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
<parent> <parent>
<groupId>org.apache.rocketmq</groupId> <groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-all</artifactId> <artifactId>rocketmq-all</artifactId>
<version>4.2.0</version> <version>4.4.0-SNAPSHOT</version>
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
...@@ -68,6 +68,10 @@ ...@@ -68,6 +68,10 @@
<groupId>org.javassist</groupId> <groupId>org.javassist</groupId>
<artifactId>javassist</artifactId> <artifactId>javassist</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
</dependencies> </dependencies>
<build> <build>
......
...@@ -18,11 +18,6 @@ package org.apache.rocketmq.broker; ...@@ -18,11 +18,6 @@ package org.apache.rocketmq.broker;
import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator; import ch.qos.logback.classic.joran.JoranConfigurator;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option; import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options; import org.apache.commons.cli.Options;
...@@ -31,6 +26,8 @@ import org.apache.rocketmq.common.BrokerConfig; ...@@ -31,6 +26,8 @@ import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MQVersion;
import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.common.RemotingUtil;
import org.apache.rocketmq.remoting.common.TlsMode; import org.apache.rocketmq.remoting.common.TlsMode;
import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyClientConfig;
...@@ -41,16 +38,21 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ...@@ -41,16 +38,21 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.srvutil.ServerUtil; import org.apache.rocketmq.srvutil.ServerUtil;
import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_ENABLE; import static org.apache.rocketmq.remoting.netty.TlsSystemConfig.TLS_ENABLE;
public class BrokerStartup { public class BrokerStartup {
public static Properties properties = null; public static Properties properties = null;
public static CommandLine commandLine = null; public static CommandLine commandLine = null;
public static String configFile = null; public static String configFile = null;
public static Logger log; public static InternalLogger log;
public static void main(String[] args) { public static void main(String[] args) {
start(createBrokerController(args)); start(createBrokerController(args));
...@@ -69,6 +71,7 @@ public class BrokerStartup { ...@@ -69,6 +71,7 @@ public class BrokerStartup {
} }
log.info(tip); log.info(tip);
System.out.printf("%s%n", tip);
return controller; return controller;
} catch (Throwable e) { } catch (Throwable e) {
e.printStackTrace(); e.printStackTrace();
...@@ -78,6 +81,12 @@ public class BrokerStartup { ...@@ -78,6 +81,12 @@ public class BrokerStartup {
return null; return null;
} }
public static void shutdown(final BrokerController controller) {
if (null != controller) {
controller.shutdown();
}
}
public static BrokerController createBrokerController(String[] args) { public static BrokerController createBrokerController(String[] args) {
System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));
...@@ -134,8 +143,7 @@ public class BrokerStartup { ...@@ -134,8 +143,7 @@ public class BrokerStartup {
MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), brokerConfig); MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), brokerConfig);
if (null == brokerConfig.getRocketmqHome()) { if (null == brokerConfig.getRocketmqHome()) {
System.out.printf("Please set the " + MixAll.ROCKETMQ_HOME_ENV System.out.printf("Please set the %s variable in your environment to match the location of the RocketMQ installation", MixAll.ROCKETMQ_HOME_ENV);
+ " variable in your environment to match the location of the RocketMQ installation");
System.exit(-2); System.exit(-2);
} }
...@@ -178,14 +186,14 @@ public class BrokerStartup { ...@@ -178,14 +186,14 @@ public class BrokerStartup {
configurator.doConfigure(brokerConfig.getRocketmqHome() + "/conf/logback_broker.xml"); configurator.doConfigure(brokerConfig.getRocketmqHome() + "/conf/logback_broker.xml");
if (commandLine.hasOption('p')) { if (commandLine.hasOption('p')) {
Logger console = LoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME); InternalLogger console = InternalLoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME);
MixAll.printObjectProperties(console, brokerConfig); MixAll.printObjectProperties(console, brokerConfig);
MixAll.printObjectProperties(console, nettyServerConfig); MixAll.printObjectProperties(console, nettyServerConfig);
MixAll.printObjectProperties(console, nettyClientConfig); MixAll.printObjectProperties(console, nettyClientConfig);
MixAll.printObjectProperties(console, messageStoreConfig); MixAll.printObjectProperties(console, messageStoreConfig);
System.exit(0); System.exit(0);
} else if (commandLine.hasOption('m')) { } else if (commandLine.hasOption('m')) {
Logger console = LoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME); InternalLogger console = InternalLoggerFactory.getLogger(LoggerName.BROKER_CONSOLE_NAME);
MixAll.printObjectProperties(console, brokerConfig, true); MixAll.printObjectProperties(console, brokerConfig, true);
MixAll.printObjectProperties(console, nettyServerConfig, true); MixAll.printObjectProperties(console, nettyServerConfig, true);
MixAll.printObjectProperties(console, nettyClientConfig, true); MixAll.printObjectProperties(console, nettyClientConfig, true);
...@@ -193,7 +201,7 @@ public class BrokerStartup { ...@@ -193,7 +201,7 @@ public class BrokerStartup {
System.exit(0); System.exit(0);
} }
log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
MixAll.printObjectProperties(log, brokerConfig); MixAll.printObjectProperties(log, brokerConfig);
MixAll.printObjectProperties(log, nettyServerConfig); MixAll.printObjectProperties(log, nettyServerConfig);
MixAll.printObjectProperties(log, nettyClientConfig); MixAll.printObjectProperties(log, nettyClientConfig);
......
...@@ -23,12 +23,12 @@ import java.util.concurrent.TimeUnit; ...@@ -23,12 +23,12 @@ import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.ChannelEventListener; import org.apache.rocketmq.remoting.ChannelEventListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ClientHousekeepingService implements ChannelEventListener { public class ClientHousekeepingService implements ChannelEventListener {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController; private final BrokerController brokerController;
private ScheduledExecutorService scheduledExecutorService = Executors private ScheduledExecutorService scheduledExecutorService = Executors
......
...@@ -26,14 +26,14 @@ import java.util.concurrent.ConcurrentHashMap; ...@@ -26,14 +26,14 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ConsumerGroupInfo { public class ConsumerGroupInfo {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final String groupName; private final String groupName;
private final ConcurrentMap<String/* Topic */, SubscriptionData> subscriptionTable = private final ConcurrentMap<String/* Topic */, SubscriptionData> subscriptionTable =
new ConcurrentHashMap<String, SubscriptionData>(); new ConcurrentHashMap<String, SubscriptionData>();
......
...@@ -25,16 +25,16 @@ import java.util.concurrent.ConcurrentHashMap; ...@@ -25,16 +25,16 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.common.RemotingUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ConsumerManager { public class ConsumerManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120;
private final ConcurrentMap<String/* Group */, ConsumerGroupInfo> consumerTable = private final ConcurrentMap<String/* Group */, ConsumerGroupInfo> consumerTable =
new ConcurrentHashMap<String, ConsumerGroupInfo>(1024); new ConcurrentHashMap<String, ConsumerGroupInfo>(1024);
......
...@@ -17,27 +17,33 @@ ...@@ -17,27 +17,33 @@
package org.apache.rocketmq.broker.client; package org.apache.rocketmq.broker.client;
import io.netty.channel.Channel; import io.netty.channel.Channel;
import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import org.apache.rocketmq.broker.util.PositiveAtomicCounter;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.common.RemotingUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ProducerManager { public class ProducerManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final long LOCK_TIMEOUT_MILLIS = 3000; private static final long LOCK_TIMEOUT_MILLIS = 3000;
private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120;
private static final int GET_AVALIABLE_CHANNEL_RETRY_COUNT = 3;
private final Lock groupChannelLock = new ReentrantLock(); private final Lock groupChannelLock = new ReentrantLock();
private final HashMap<String /* group name */, HashMap<Channel, ClientChannelInfo>> groupChannelTable = private final HashMap<String /* group name */, HashMap<Channel, ClientChannelInfo>> groupChannelTable =
new HashMap<String, HashMap<Channel, ClientChannelInfo>>(); new HashMap<String, HashMap<Channel, ClientChannelInfo>>();
private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter();
public ProducerManager() { public ProducerManager() {
} }
...@@ -185,4 +191,36 @@ public class ProducerManager { ...@@ -185,4 +191,36 @@ public class ProducerManager {
log.error("", e); log.error("", e);
} }
} }
public Channel getAvaliableChannel(String groupId) {
HashMap<Channel, ClientChannelInfo> channelClientChannelInfoHashMap = groupChannelTable.get(groupId);
List<Channel> channelList = new ArrayList<Channel>();
if (channelClientChannelInfoHashMap != null) {
for (Channel channel : channelClientChannelInfoHashMap.keySet()) {
channelList.add(channel);
}
int size = channelList.size();
if (0 == size) {
log.warn("Channel list is empty. groupId={}", groupId);
return null;
}
int index = positiveAtomicCounter.incrementAndGet() % size;
Channel channel = channelList.get(index);
int count = 0;
boolean isOk = channel.isActive() && channel.isWritable();
while (count++ < GET_AVALIABLE_CHANNEL_RETRY_COUNT) {
if (isOk) {
return channel;
}
index = (++index) % size;
channel = channelList.get(index);
isOk = channel.isActive() && channel.isWritable();
}
} else {
log.warn("Check transaction failed, channel table is empty. groupId={}", groupId);
return null;
}
return null;
}
} }
...@@ -17,23 +17,15 @@ ...@@ -17,23 +17,15 @@
package org.apache.rocketmq.broker.client.net; package org.apache.rocketmq.broker.client.net;
import io.netty.channel.Channel; import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.FileRegion;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo; import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.broker.pagecache.OneMessageTransfer;
import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MQVersion;
import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.message.MessageQueueForC; import org.apache.rocketmq.common.message.MessageQueueForC;
import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.RequestCode;
...@@ -45,16 +37,22 @@ import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHe ...@@ -45,16 +37,22 @@ import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHe
import org.apache.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; import org.apache.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader;
import org.apache.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; import org.apache.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader;
import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.slf4j.Logger; import java.util.ArrayList;
import org.slf4j.LoggerFactory; import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentMap;
public class Broker2Client { public class Broker2Client {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController; private final BrokerController brokerController;
public Broker2Client(BrokerController brokerController) { public Broker2Client(BrokerController brokerController) {
...@@ -62,34 +60,22 @@ public class Broker2Client { ...@@ -62,34 +60,22 @@ public class Broker2Client {
} }
public void checkProducerTransactionState( public void checkProducerTransactionState(
final String group,
final Channel channel, final Channel channel,
final CheckTransactionStateRequestHeader requestHeader, final CheckTransactionStateRequestHeader requestHeader,
final SelectMappedBufferResult selectMappedBufferResult) { final MessageExt messageExt) throws Exception {
RemotingCommand request = RemotingCommand request =
RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, requestHeader); RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, requestHeader);
request.markOnewayRPC(); request.setBody(MessageDecoder.encode(messageExt, false));
try { try {
FileRegion fileRegion = this.brokerController.getRemotingServer().invokeOneway(channel, request, 10);
new OneMessageTransfer(request.encodeHeader(selectMappedBufferResult.getSize()), } catch (Exception e) {
selectMappedBufferResult); log.error("Check transaction failed because invoke producer exception. group={}, msgId={}", group, messageExt.getMsgId(), e.getMessage());
channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
selectMappedBufferResult.release();
if (!future.isSuccess()) {
log.error("invokeProducer failed,", future.cause());
}
}
});
} catch (Throwable e) {
log.error("invokeProducer exception", e);
selectMappedBufferResult.release();
} }
} }
public RemotingCommand callClient(final Channel channel, public RemotingCommand callClient(final Channel channel,
final RemotingCommand request final RemotingCommand request
) throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException { ) throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException {
return this.brokerController.getRemotingServer().invokeSync(channel, request, 10000); return this.brokerController.getRemotingServer().invokeSync(channel, request, 10000);
} }
...@@ -119,7 +105,7 @@ public class Broker2Client { ...@@ -119,7 +105,7 @@ public class Broker2Client {
} }
public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce, public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce,
boolean isC) { boolean isC) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null); final RemotingCommand response = RemotingCommand.createResponseCommand(null);
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
......
...@@ -23,12 +23,12 @@ import java.util.concurrent.ConcurrentMap; ...@@ -23,12 +23,12 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RebalanceLockManager { public class RebalanceLockManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.REBALANCE_LOCK_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.REBALANCE_LOCK_LOGGER_NAME);
private final static long REBALANCE_LOCK_MAX_LIVE_TIME = Long.parseLong(System.getProperty( private final static long REBALANCE_LOCK_MAX_LIVE_TIME = Long.parseLong(System.getProperty(
"rocketmq.broker.rebalance.lockMaxLiveTime", "60000")); "rocketmq.broker.rebalance.lockMaxLiveTime", "60000"));
private final Lock lock = new ReentrantLock(); private final Lock lock = new ReentrantLock();
......
...@@ -19,11 +19,11 @@ package org.apache.rocketmq.broker.filter; ...@@ -19,11 +19,11 @@ package org.apache.rocketmq.broker.filter;
import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.filter.util.BitsArray; import org.apache.rocketmq.filter.util.BitsArray;
import org.apache.rocketmq.store.CommitLogDispatcher; import org.apache.rocketmq.store.CommitLogDispatcher;
import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.DispatchRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection; import java.util.Collection;
import java.util.Iterator; import java.util.Iterator;
...@@ -33,7 +33,7 @@ import java.util.Iterator; ...@@ -33,7 +33,7 @@ import java.util.Iterator;
*/ */
public class CommitLogDispatcherCalcBitMap implements CommitLogDispatcher { public class CommitLogDispatcherCalcBitMap implements CommitLogDispatcher {
private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME);
protected final BrokerConfig brokerConfig; protected final BrokerConfig brokerConfig;
protected final ConsumerFilterManager consumerFilterManager; protected final ConsumerFilterManager consumerFilterManager;
......
...@@ -22,14 +22,14 @@ import org.apache.rocketmq.broker.BrokerController; ...@@ -22,14 +22,14 @@ import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.BrokerPathConfigHelper; import org.apache.rocketmq.broker.BrokerPathConfigHelper;
import org.apache.rocketmq.common.ConfigManager; import org.apache.rocketmq.common.ConfigManager;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.filter.FilterFactory; import org.apache.rocketmq.filter.FilterFactory;
import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.ExpressionType;
import org.apache.rocketmq.filter.util.BloomFilter; import org.apache.rocketmq.filter.util.BloomFilter;
import org.apache.rocketmq.filter.util.BloomFilterData; import org.apache.rocketmq.filter.util.BloomFilterData;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection; import java.util.Collection;
import java.util.HashSet; import java.util.HashSet;
...@@ -42,7 +42,7 @@ import java.util.concurrent.ConcurrentHashMap; ...@@ -42,7 +42,7 @@ import java.util.concurrent.ConcurrentHashMap;
*/ */
public class ConsumerFilterManager extends ConfigManager { public class ConsumerFilterManager extends ConfigManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME);
private static final long MS_24_HOUR = 24 * 3600 * 1000; private static final long MS_24_HOUR = 24 * 3600 * 1000;
......
...@@ -19,21 +19,21 @@ package org.apache.rocketmq.broker.filter; ...@@ -19,21 +19,21 @@ package org.apache.rocketmq.broker.filter;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.ExpressionType;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.filter.util.BitsArray; import org.apache.rocketmq.filter.util.BitsArray;
import org.apache.rocketmq.filter.util.BloomFilter; import org.apache.rocketmq.filter.util.BloomFilter;
import org.apache.rocketmq.store.ConsumeQueueExt; import org.apache.rocketmq.store.ConsumeQueueExt;
import org.apache.rocketmq.store.MessageFilter; import org.apache.rocketmq.store.MessageFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.Map; import java.util.Map;
public class ExpressionMessageFilter implements MessageFilter { public class ExpressionMessageFilter implements MessageFilter {
protected static final Logger log = LoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME); protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.FILTER_LOGGER_NAME);
protected final SubscriptionData subscriptionData; protected final SubscriptionData subscriptionData;
protected final ConsumerFilterData consumerFilterData; protected final ConsumerFilterData consumerFilterData;
......
...@@ -31,14 +31,14 @@ import org.apache.rocketmq.broker.BrokerController; ...@@ -31,14 +31,14 @@ import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.BrokerStartup; import org.apache.rocketmq.broker.BrokerStartup;
import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.common.RemotingUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FilterServerManager { public class FilterServerManager {
public static final long FILTER_SERVER_MAX_IDLE_TIME_MILLS = 30000; public static final long FILTER_SERVER_MAX_IDLE_TIME_MILLS = 30000;
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final ConcurrentMap<Channel, FilterServerInfo> filterServerTable = private final ConcurrentMap<Channel, FilterServerInfo> filterServerTable =
new ConcurrentHashMap<Channel, FilterServerInfo>(16); new ConcurrentHashMap<Channel, FilterServerInfo>(16);
private final BrokerController brokerController; private final BrokerController brokerController;
......
...@@ -17,10 +17,11 @@ ...@@ -17,10 +17,11 @@
package org.apache.rocketmq.broker.filtersrv; package org.apache.rocketmq.broker.filtersrv;
import org.slf4j.Logger;
import org.apache.rocketmq.logging.InternalLogger;
public class FilterServerUtil { public class FilterServerUtil {
public static void callShell(final String shellString, final Logger log) { public static void callShell(final String shellString, final InternalLogger log) {
Process process = null; Process process = null;
try { try {
String[] cmdArray = splitShellString(shellString); String[] cmdArray = splitShellString(shellString);
......
...@@ -23,17 +23,17 @@ import java.util.concurrent.TimeUnit; ...@@ -23,17 +23,17 @@ import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.netty.RequestTask; import org.apache.rocketmq.remoting.netty.RequestTask;
import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* BrokerFastFailure will cover {@link BrokerController#sendThreadPoolQueue} and * BrokerFastFailure will cover {@link BrokerController#sendThreadPoolQueue} and
* {@link BrokerController#pullThreadPoolQueue} * {@link BrokerController#pullThreadPoolQueue}
*/ */
public class BrokerFastFailure { public class BrokerFastFailure {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(
"BrokerFastFailureScheduledThread")); "BrokerFastFailureScheduledThread"));
private final BrokerController brokerController; private final BrokerController brokerController;
...@@ -44,8 +44,10 @@ public class BrokerFastFailure { ...@@ -44,8 +44,10 @@ public class BrokerFastFailure {
public static RequestTask castRunnable(final Runnable runnable) { public static RequestTask castRunnable(final Runnable runnable) {
try { try {
FutureTaskExt object = (FutureTaskExt) runnable; if (runnable instanceof FutureTaskExt) {
return (RequestTask) object.getRunnable(); FutureTaskExt object = (FutureTaskExt) runnable;
return (RequestTask) object.getRunnable();
}
} catch (Throwable e) { } catch (Throwable e) {
log.error(String.format("castRunnable exception, %s", runnable.getClass().getName()), e); log.error(String.format("castRunnable exception, %s", runnable.getClass().getName()), e);
} }
...@@ -87,6 +89,9 @@ public class BrokerFastFailure { ...@@ -87,6 +89,9 @@ public class BrokerFastFailure {
cleanExpiredRequestInQueue(this.brokerController.getPullThreadPoolQueue(), cleanExpiredRequestInQueue(this.brokerController.getPullThreadPoolQueue(),
this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue()); this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue());
cleanExpiredRequestInQueue(this.brokerController.getHeartbeatThreadPoolQueue(),
this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue());
} }
void cleanExpiredRequestInQueue(final BlockingQueue<Runnable> blockingQueue, final long maxWaitTimeMillsInQueue) { void cleanExpiredRequestInQueue(final BlockingQueue<Runnable> blockingQueue, final long maxWaitTimeMillsInQueue) {
......
...@@ -25,12 +25,12 @@ import org.apache.rocketmq.broker.BrokerController; ...@@ -25,12 +25,12 @@ import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.ServiceThread;
import org.apache.rocketmq.common.SystemClock; import org.apache.rocketmq.common.SystemClock;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.store.ConsumeQueueExt; import org.apache.rocketmq.store.ConsumeQueueExt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PullRequestHoldService extends ServiceThread { public class PullRequestHoldService extends ServiceThread {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final String TOPIC_QUEUEID_SEPARATOR = "@"; private static final String TOPIC_QUEUEID_SEPARATOR = "@";
private final BrokerController brokerController; private final BrokerController brokerController;
private final SystemClock systemClock = new SystemClock(); private final SystemClock systemClock = new SystemClock();
......
...@@ -29,12 +29,12 @@ import org.apache.rocketmq.broker.BrokerPathConfigHelper; ...@@ -29,12 +29,12 @@ import org.apache.rocketmq.broker.BrokerPathConfigHelper;
import org.apache.rocketmq.common.ConfigManager; import org.apache.rocketmq.common.ConfigManager;
import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ConsumerOffsetManager extends ConfigManager { public class ConsumerOffsetManager extends ConfigManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final String TOPIC_GROUP_SEPARATOR = "@"; private static final String TOPIC_GROUP_SEPARATOR = "@";
private ConcurrentMap<String/* topic@group */, ConcurrentMap<Integer, Long>> offsetTable = private ConcurrentMap<String/* topic@group */, ConcurrentMap<Integer, Long>> offsetTable =
......
...@@ -16,12 +16,23 @@ ...@@ -16,12 +16,23 @@
*/ */
package org.apache.rocketmq.broker.out; package org.apache.rocketmq.broker.out;
import com.google.common.collect.Lists;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor;
import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult;
import org.apache.rocketmq.common.namesrv.TopAddressing; import org.apache.rocketmq.common.namesrv.TopAddressing;
import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.RequestCode;
...@@ -31,6 +42,8 @@ import org.apache.rocketmq.common.protocol.body.KVTable; ...@@ -31,6 +42,8 @@ import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody; import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionRequestHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionResponseHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader;
...@@ -44,14 +57,14 @@ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ...@@ -44,14 +57,14 @@ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BrokerOuterAPI { public class BrokerOuterAPI {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final RemotingClient remotingClient; private final RemotingClient remotingClient;
private final TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr()); private final TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr());
private String nameSrvAddr = null; private String nameSrvAddr = null;
private BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES,
new ArrayBlockingQueue<Runnable>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true));
public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) { public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) {
this(nettyClientConfig, null); this(nettyClientConfig, null);
...@@ -68,6 +81,7 @@ public class BrokerOuterAPI { ...@@ -68,6 +81,7 @@ public class BrokerOuterAPI {
public void shutdown() { public void shutdown() {
this.remotingClient.shutdown(); this.remotingClient.shutdown();
this.brokerOuterExecutor.shutdown();
} }
public String fetchNameServerAddr() { public String fetchNameServerAddr() {
...@@ -97,7 +111,7 @@ public class BrokerOuterAPI { ...@@ -97,7 +111,7 @@ public class BrokerOuterAPI {
this.remotingClient.updateNameServerAddressList(lst); this.remotingClient.updateNameServerAddressList(lst);
} }
public RegisterBrokerResult registerBrokerAll( public List<RegisterBrokerResult> registerBrokerAll(
final String clusterName, final String clusterName,
final String brokerAddr, final String brokerAddr,
final String brokerName, final String brokerName,
...@@ -106,54 +120,67 @@ public class BrokerOuterAPI { ...@@ -106,54 +120,67 @@ public class BrokerOuterAPI {
final TopicConfigSerializeWrapper topicConfigWrapper, final TopicConfigSerializeWrapper topicConfigWrapper,
final List<String> filterServerList, final List<String> filterServerList,
final boolean oneway, final boolean oneway,
final int timeoutMills) { final int timeoutMills,
RegisterBrokerResult registerBrokerResult = null; final boolean compressed) {
final List<RegisterBrokerResult> registerBrokerResultList = Lists.newArrayList();
List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList(); List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();
if (nameServerAddressList != null) { if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
for (String namesrvAddr : nameServerAddressList) {
try { final RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader();
RegisterBrokerResult result = this.registerBroker(namesrvAddr, clusterName, brokerAddr, brokerName, brokerId, requestHeader.setBrokerAddr(brokerAddr);
haServerAddr, topicConfigWrapper, filterServerList, oneway, timeoutMills); requestHeader.setBrokerId(brokerId);
if (result != null) { requestHeader.setBrokerName(brokerName);
registerBrokerResult = result; requestHeader.setClusterName(clusterName);
requestHeader.setHaServerAddr(haServerAddr);
requestHeader.setCompressed(compressed);
RegisterBrokerBody requestBody = new RegisterBrokerBody();
requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper);
requestBody.setFilterServerList(filterServerList);
final byte[] body = requestBody.encode(compressed);
final int bodyCrc32 = UtilAll.crc32(body);
requestHeader.setBodyCrc32(bodyCrc32);
final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size());
for (final String namesrvAddr : nameServerAddressList) {
brokerOuterExecutor.execute(new Runnable() {
@Override
public void run() {
try {
RegisterBrokerResult result = registerBroker(namesrvAddr,oneway, timeoutMills,requestHeader,body);
if (result != null) {
registerBrokerResultList.add(result);
}
log.info("register broker to name server {} OK", namesrvAddr);
} catch (Exception e) {
log.warn("registerBroker Exception, {}", namesrvAddr, e);
} finally {
countDownLatch.countDown();
}
} }
});
}
log.info("register broker to name server {} OK", namesrvAddr); try {
} catch (Exception e) { countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS);
log.warn("registerBroker Exception, {}", namesrvAddr, e); } catch (InterruptedException e) {
}
} }
} }
return registerBrokerResult; return registerBrokerResultList;
} }
private RegisterBrokerResult registerBroker( private RegisterBrokerResult registerBroker(
final String namesrvAddr, final String namesrvAddr,
final String clusterName,
final String brokerAddr,
final String brokerName,
final long brokerId,
final String haServerAddr,
final TopicConfigSerializeWrapper topicConfigWrapper,
final List<String> filterServerList,
final boolean oneway, final boolean oneway,
final int timeoutMills final int timeoutMills,
final RegisterBrokerRequestHeader requestHeader,
final byte[] body
) throws RemotingCommandException, MQBrokerException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, ) throws RemotingCommandException, MQBrokerException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException,
InterruptedException { InterruptedException {
RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader();
requestHeader.setBrokerAddr(brokerAddr);
requestHeader.setBrokerId(brokerId);
requestHeader.setBrokerName(brokerName);
requestHeader.setClusterName(clusterName);
requestHeader.setHaServerAddr(haServerAddr);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REGISTER_BROKER, requestHeader); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REGISTER_BROKER, requestHeader);
request.setBody(body);
RegisterBrokerBody requestBody = new RegisterBrokerBody();
requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper);
requestBody.setFilterServerList(filterServerList);
request.setBody(requestBody.encode());
if (oneway) { if (oneway) {
try { try {
...@@ -231,6 +258,71 @@ public class BrokerOuterAPI { ...@@ -231,6 +258,71 @@ public class BrokerOuterAPI {
throw new MQBrokerException(response.getCode(), response.getRemark()); throw new MQBrokerException(response.getCode(), response.getRemark());
} }
public List<Boolean> needRegister(
final String clusterName,
final String brokerAddr,
final String brokerName,
final long brokerId,
final TopicConfigSerializeWrapper topicConfigWrapper,
final int timeoutMills) {
final List<Boolean> changedList = new CopyOnWriteArrayList<>();
List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();
if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size());
for (final String namesrvAddr : nameServerAddressList) {
brokerOuterExecutor.execute(new Runnable() {
@Override
public void run() {
try {
QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();
requestHeader.setBrokerAddr(brokerAddr);
requestHeader.setBrokerId(brokerId);
requestHeader.setBrokerName(brokerName);
requestHeader.setClusterName(clusterName);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_DATA_VERSION, requestHeader);
request.setBody(topicConfigWrapper.getDataVersion().encode());
RemotingCommand response = remotingClient.invokeSync(namesrvAddr, request, timeoutMills);
DataVersion nameServerDataVersion = null;
Boolean changed = false;
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
QueryDataVersionResponseHeader queryDataVersionResponseHeader =
(QueryDataVersionResponseHeader) response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class);
changed = queryDataVersionResponseHeader.getChanged();
byte[] body = response.getBody();
if (body != null) {
nameServerDataVersion = DataVersion.decode(body, DataVersion.class);
if (!topicConfigWrapper.getDataVersion().equals(nameServerDataVersion)) {
changed = true;
}
}
if (changed == null || changed) {
changedList.add(Boolean.TRUE);
}
}
default:
break;
}
log.warn("Query data version from name server {} OK,changed {}, broker {},name server {}", namesrvAddr, changed, topicConfigWrapper.getDataVersion(), nameServerDataVersion == null ? "" : nameServerDataVersion);
} catch (Exception e) {
changedList.add(Boolean.TRUE);
log.error("Query data version from name server {} Exception, {}", namesrvAddr, e);
} finally {
countDownLatch.countDown();
}
}
});
}
try {
countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
log.error("query dataversion from nameserver countDownLatch await Exception", e);
}
}
return changedList;
}
public TopicConfigSerializeWrapper getAllTopicConfig( public TopicConfigSerializeWrapper getAllTopicConfig(
final String addr) throws RemotingConnectException, RemotingSendRequestException, final String addr) throws RemotingConnectException, RemotingSendRequestException,
RemotingTimeoutException, InterruptedException, MQBrokerException { RemotingTimeoutException, InterruptedException, MQBrokerException {
......
...@@ -27,6 +27,8 @@ import org.apache.rocketmq.common.constant.DBMsgConstants; ...@@ -27,6 +27,8 @@ import org.apache.rocketmq.common.constant.DBMsgConstants;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageDecoder;
...@@ -43,8 +45,6 @@ import org.apache.rocketmq.remoting.exception.RemotingCommandException; ...@@ -43,8 +45,6 @@ import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.net.SocketAddress; import java.net.SocketAddress;
...@@ -53,7 +53,7 @@ import java.util.Map; ...@@ -53,7 +53,7 @@ import java.util.Map;
import java.util.Random; import java.util.Random;
public abstract class AbstractSendMessageProcessor implements NettyRequestProcessor { public abstract class AbstractSendMessageProcessor implements NettyRequestProcessor {
protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
protected final static int DLQ_NUMS_PER_GROUP = 1; protected final static int DLQ_NUMS_PER_GROUP = 1;
protected final BrokerController brokerController; protected final BrokerController brokerController;
......
...@@ -44,6 +44,8 @@ import org.apache.rocketmq.common.admin.OffsetWrapper; ...@@ -44,6 +44,8 @@ import org.apache.rocketmq.common.admin.OffsetWrapper;
import org.apache.rocketmq.common.admin.TopicOffset; import org.apache.rocketmq.common.admin.TopicOffset;
import org.apache.rocketmq.common.admin.TopicStatsTable; import org.apache.rocketmq.common.admin.TopicStatsTable;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageId;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
...@@ -113,11 +115,9 @@ import org.apache.rocketmq.store.ConsumeQueueExt; ...@@ -113,11 +115,9 @@ import org.apache.rocketmq.store.ConsumeQueueExt;
import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.MessageFilter; import org.apache.rocketmq.store.MessageFilter;
import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class AdminBrokerProcessor implements NettyRequestProcessor { public class AdminBrokerProcessor implements NettyRequestProcessor {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController; private final BrokerController brokerController;
public AdminBrokerProcessor(final BrokerController brokerController) { public AdminBrokerProcessor(final BrokerController brokerController) {
...@@ -212,7 +212,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ...@@ -212,7 +212,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
return false; return false;
} }
private RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException { RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null); final RemotingCommand response = RemotingCommand.createResponseCommand(null);
final CreateTopicRequestHeader requestHeader = final CreateTopicRequestHeader requestHeader =
...@@ -245,12 +245,13 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ...@@ -245,12 +245,13 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag()); topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag());
this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);
this.brokerController.registerBrokerAll(false, true);
this.brokerController.registerIncrementBrokerData(topicConfig,this.brokerController.getTopicConfigManager().getDataVersion());
return null; return null;
} }
private RemotingCommand deleteTopic(ChannelHandlerContext ctx, private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException { RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null); final RemotingCommand response = RemotingCommand.createResponseCommand(null);
DeleteTopicRequestHeader requestHeader = DeleteTopicRequestHeader requestHeader =
...@@ -296,7 +297,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ...@@ -296,7 +297,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
return response; return response;
} }
private RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) { private synchronized RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null); final RemotingCommand response = RemotingCommand.createResponseCommand(null);
log.info("updateBrokerConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); log.info("updateBrokerConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
...@@ -310,8 +311,8 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ...@@ -310,8 +311,8 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
log.info("updateBrokerConfig, new config: [{}] client: {} ", properties, ctx.channel().remoteAddress()); log.info("updateBrokerConfig, new config: [{}] client: {} ", properties, ctx.channel().remoteAddress());
this.brokerController.getConfiguration().update(properties); this.brokerController.getConfiguration().update(properties);
if (properties.containsKey("brokerPermission")) { if (properties.containsKey("brokerPermission")) {
this.brokerController.registerBrokerAll(false, false);
this.brokerController.getTopicConfigManager().getDataVersion().nextVersion(); this.brokerController.getTopicConfigManager().getDataVersion().nextVersion();
this.brokerController.registerBrokerAll(false, false, true);
} }
} else { } else {
log.error("string2Properties error"); log.error("string2Properties error");
......
...@@ -23,6 +23,7 @@ import org.apache.rocketmq.common.MixAll; ...@@ -23,6 +23,7 @@ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.ExpressionType;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.body.CheckClientRequestBody; import org.apache.rocketmq.common.protocol.body.CheckClientRequestBody;
...@@ -35,15 +36,14 @@ import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; ...@@ -35,15 +36,14 @@ import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.common.sysflag.TopicSysFlag; import org.apache.rocketmq.common.sysflag.TopicSysFlag;
import org.apache.rocketmq.filter.FilterFactory; import org.apache.rocketmq.filter.FilterFactory;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ClientManageProcessor implements NettyRequestProcessor { public class ClientManageProcessor implements NettyRequestProcessor {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController; private final BrokerController brokerController;
public ClientManageProcessor(final BrokerController brokerController) { public ClientManageProcessor(final BrokerController brokerController) {
......
...@@ -21,6 +21,7 @@ import java.util.List; ...@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo; import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader;
...@@ -30,15 +31,14 @@ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHead ...@@ -30,15 +31,14 @@ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHead
import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader; import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader;
import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader;
import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetResponseHeader; import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetResponseHeader;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ConsumerManageProcessor implements NettyRequestProcessor { public class ConsumerManageProcessor implements NettyRequestProcessor {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController; private final BrokerController brokerController;
......
...@@ -18,6 +18,7 @@ package org.apache.rocketmq.broker.processor; ...@@ -18,6 +18,7 @@ package org.apache.rocketmq.broker.processor;
import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelHandlerContext;
import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.transaction.OperationResult;
import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.TopicFilterType;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageAccessor;
...@@ -27,18 +28,21 @@ import org.apache.rocketmq.common.message.MessageExt; ...@@ -27,18 +28,21 @@ import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageResult;
import org.slf4j.Logger; import org.apache.rocketmq.store.config.BrokerRole;
import org.slf4j.LoggerFactory;
/**
* EndTransaction processor: process commit and rollback message
*/
public class EndTransactionProcessor implements NettyRequestProcessor { public class EndTransactionProcessor implements NettyRequestProcessor {
private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
private final BrokerController brokerController; private final BrokerController brokerController;
public EndTransactionProcessor(final BrokerController brokerController) { public EndTransactionProcessor(final BrokerController brokerController) {
...@@ -46,16 +50,22 @@ public class EndTransactionProcessor implements NettyRequestProcessor { ...@@ -46,16 +50,22 @@ public class EndTransactionProcessor implements NettyRequestProcessor {
} }
@Override @Override
public RemotingCommand processRequest(ChannelHandlerContext ctx, public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws
RemotingCommand request) throws RemotingCommandException { RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null); final RemotingCommand response = RemotingCommand.createResponseCommand(null);
final EndTransactionRequestHeader requestHeader = final EndTransactionRequestHeader requestHeader =
(EndTransactionRequestHeader) request.decodeCommandCustomHeader(EndTransactionRequestHeader.class); (EndTransactionRequestHeader)request.decodeCommandCustomHeader(EndTransactionRequestHeader.class);
LOGGER.info("Transaction request:{}", requestHeader);
if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) {
response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE);
LOGGER.warn("Message store is slave mode, so end transaction is forbidden. ");
return response;
}
if (requestHeader.getFromTransactionCheck()) { if (requestHeader.getFromTransactionCheck()) {
switch (requestHeader.getCommitOrRollback()) { switch (requestHeader.getCommitOrRollback()) {
case MessageSysFlag.TRANSACTION_NOT_TYPE: { case MessageSysFlag.TRANSACTION_NOT_TYPE: {
LOGGER.warn("check producer[{}] transaction state, but it's pending status." LOGGER.warn("Check producer[{}] transaction state, but it's pending status."
+ "RequestHeader: {} Remark: {}", + "RequestHeader: {} Remark: {}",
RemotingHelper.parseChannelRemoteAddr(ctx.channel()), RemotingHelper.parseChannelRemoteAddr(ctx.channel()),
requestHeader.toString(), requestHeader.toString(),
...@@ -64,7 +74,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor { ...@@ -64,7 +74,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor {
} }
case MessageSysFlag.TRANSACTION_COMMIT_TYPE: { case MessageSysFlag.TRANSACTION_COMMIT_TYPE: {
LOGGER.warn("check producer[{}] transaction state, the producer commit the message." LOGGER.warn("Check producer[{}] transaction state, the producer commit the message."
+ "RequestHeader: {} Remark: {}", + "RequestHeader: {} Remark: {}",
RemotingHelper.parseChannelRemoteAddr(ctx.channel()), RemotingHelper.parseChannelRemoteAddr(ctx.channel()),
requestHeader.toString(), requestHeader.toString(),
...@@ -74,7 +84,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor { ...@@ -74,7 +84,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor {
} }
case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: { case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: {
LOGGER.warn("check producer[{}] transaction state, the producer rollback the message." LOGGER.warn("Check producer[{}] transaction state, the producer rollback the message."
+ "RequestHeader: {} Remark: {}", + "RequestHeader: {} Remark: {}",
RemotingHelper.parseChannelRemoteAddr(ctx.channel()), RemotingHelper.parseChannelRemoteAddr(ctx.channel()),
requestHeader.toString(), requestHeader.toString(),
...@@ -87,7 +97,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor { ...@@ -87,7 +97,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor {
} else { } else {
switch (requestHeader.getCommitOrRollback()) { switch (requestHeader.getCommitOrRollback()) {
case MessageSysFlag.TRANSACTION_NOT_TYPE: { case MessageSysFlag.TRANSACTION_NOT_TYPE: {
LOGGER.warn("the producer[{}] end transaction in sending message, and it's pending status." LOGGER.warn("The producer[{}] end transaction in sending message, and it's pending status."
+ "RequestHeader: {} Remark: {}", + "RequestHeader: {} Remark: {}",
RemotingHelper.parseChannelRemoteAddr(ctx.channel()), RemotingHelper.parseChannelRemoteAddr(ctx.channel()),
requestHeader.toString(), requestHeader.toString(),
...@@ -100,7 +110,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor { ...@@ -100,7 +110,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor {
} }
case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: { case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: {
LOGGER.warn("the producer[{}] end transaction in sending message, rollback the message." LOGGER.warn("The producer[{}] end transaction in sending message, rollback the message."
+ "RequestHeader: {} Remark: {}", + "RequestHeader: {} Remark: {}",
RemotingHelper.parseChannelRemoteAddr(ctx.channel()), RemotingHelper.parseChannelRemoteAddr(ctx.channel()),
requestHeader.toString(), requestHeader.toString(),
...@@ -111,122 +121,145 @@ public class EndTransactionProcessor implements NettyRequestProcessor { ...@@ -111,122 +121,145 @@ public class EndTransactionProcessor implements NettyRequestProcessor {
return null; return null;
} }
} }
OperationResult result = new OperationResult();
if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == requestHeader.getCommitOrRollback()) {
result = this.brokerController.getTransactionalMessageService().commitMessage(requestHeader);
if (result.getResponseCode() == ResponseCode.SUCCESS) {
RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader);
if (res.getCode() == ResponseCode.SUCCESS) {
MessageExtBrokerInner msgInner = endMessageTransaction(result.getPrepareMessage());
msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback()));
msgInner.setQueueOffset(requestHeader.getTranStateTableOffset());
msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset());
msgInner.setStoreTimestamp(result.getPrepareMessage().getStoreTimestamp());
RemotingCommand sendResult = sendFinalMessage(msgInner);
if (sendResult.getCode() == ResponseCode.SUCCESS) {
this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage());
}
return sendResult;
}
return res;
}
} else if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) {
result = this.brokerController.getTransactionalMessageService().rollbackMessage(requestHeader);
if (result.getResponseCode() == ResponseCode.SUCCESS) {
RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader);
if (res.getCode() == ResponseCode.SUCCESS) {
this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage());
}
return res;
}
}
response.setCode(result.getResponseCode());
response.setRemark(result.getResponseRemark());
return response;
}
@Override
public boolean rejectRequest() {
return false;
}
final MessageExt msgExt = this.brokerController.getMessageStore().lookMessageByOffset(requestHeader.getCommitLogOffset()); private RemotingCommand checkPrepareMessage(MessageExt msgExt, EndTransactionRequestHeader requestHeader) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
if (msgExt != null) { if (msgExt != null) {
final String pgroupRead = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); final String pgroupRead = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);
if (!pgroupRead.equals(requestHeader.getProducerGroup())) { if (!pgroupRead.equals(requestHeader.getProducerGroup())) {
response.setCode(ResponseCode.SYSTEM_ERROR); response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("the producer group wrong"); response.setRemark("The producer group wrong");
return response; return response;
} }
if (msgExt.getQueueOffset() != requestHeader.getTranStateTableOffset()) { if (msgExt.getQueueOffset() != requestHeader.getTranStateTableOffset()) {
response.setCode(ResponseCode.SYSTEM_ERROR); response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("the transaction state table offset wrong"); response.setRemark("The transaction state table offset wrong");
return response; return response;
} }
if (msgExt.getCommitLogOffset() != requestHeader.getCommitLogOffset()) { if (msgExt.getCommitLogOffset() != requestHeader.getCommitLogOffset()) {
response.setCode(ResponseCode.SYSTEM_ERROR); response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("the commit log offset wrong"); response.setRemark("The commit log offset wrong");
return response;
}
MessageExtBrokerInner msgInner = this.endMessageTransaction(msgExt);
msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback()));
msgInner.setQueueOffset(requestHeader.getTranStateTableOffset());
msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset());
msgInner.setStoreTimestamp(msgExt.getStoreTimestamp());
if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) {
msgInner.setBody(null);
}
final MessageStore messageStore = this.brokerController.getMessageStore();
final PutMessageResult putMessageResult = messageStore.putMessage(msgInner);
if (putMessageResult != null) {
switch (putMessageResult.getPutMessageStatus()) {
// Success
case PUT_OK:
case FLUSH_DISK_TIMEOUT:
case FLUSH_SLAVE_TIMEOUT:
case SLAVE_NOT_AVAILABLE:
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
break;
// Failed
case CREATE_MAPEDFILE_FAILED:
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("create mapped file failed.");
break;
case MESSAGE_ILLEGAL:
case PROPERTIES_SIZE_EXCEEDED:
response.setCode(ResponseCode.MESSAGE_ILLEGAL);
response.setRemark("the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k.");
break;
case SERVICE_NOT_AVAILABLE:
response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);
response.setRemark("service not available now.");
break;
case OS_PAGECACHE_BUSY:
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("OS page cache busy, please try another machine");
break;
case UNKNOWN_ERROR:
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("UNKNOWN_ERROR");
break;
default:
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("UNKNOWN_ERROR DEFAULT");
break;
}
return response; return response;
} else {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("store putMessage return null");
} }
} else { } else {
response.setCode(ResponseCode.SYSTEM_ERROR); response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("find prepared transaction message failed"); response.setRemark("Find prepared transaction message failed");
return response; return response;
} }
response.setCode(ResponseCode.SUCCESS);
return response; return response;
} }
@Override
public boolean rejectRequest() {
return false;
}
private MessageExtBrokerInner endMessageTransaction(MessageExt msgExt) { private MessageExtBrokerInner endMessageTransaction(MessageExt msgExt) {
MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
msgInner.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC));
msgInner.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID)));
msgInner.setBody(msgExt.getBody()); msgInner.setBody(msgExt.getBody());
msgInner.setFlag(msgExt.getFlag()); msgInner.setFlag(msgExt.getFlag());
MessageAccessor.setProperties(msgInner, msgExt.getProperties()); msgInner.setBornTimestamp(msgExt.getBornTimestamp());
msgInner.setBornHost(msgExt.getBornHost());
msgInner.setStoreHost(msgExt.getStoreHost());
msgInner.setReconsumeTimes(msgExt.getReconsumeTimes());
msgInner.setWaitStoreMsgOK(false);
msgInner.setTransactionId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
msgInner.setSysFlag(msgExt.getSysFlag());
TopicFilterType topicFilterType = TopicFilterType topicFilterType =
(msgInner.getSysFlag() & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG ? TopicFilterType.MULTI_TAG (msgInner.getSysFlag() & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG ? TopicFilterType.MULTI_TAG
: TopicFilterType.SINGLE_TAG; : TopicFilterType.SINGLE_TAG;
long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags());
msgInner.setTagsCode(tagsCodeValue); msgInner.setTagsCode(tagsCodeValue);
MessageAccessor.setProperties(msgInner, msgExt.getProperties());
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));
MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC);
msgInner.setSysFlag(msgExt.getSysFlag()); MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID);
msgInner.setBornTimestamp(msgExt.getBornTimestamp());
msgInner.setBornHost(msgExt.getBornHost());
msgInner.setStoreHost(msgExt.getStoreHost());
msgInner.setReconsumeTimes(msgExt.getReconsumeTimes());
msgInner.setWaitStoreMsgOK(false);
MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_DELAY_TIME_LEVEL);
msgInner.setTopic(msgExt.getTopic());
msgInner.setQueueId(msgExt.getQueueId());
return msgInner; return msgInner;
} }
private RemotingCommand sendFinalMessage(MessageExtBrokerInner msgInner) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
final PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
if (putMessageResult != null) {
switch (putMessageResult.getPutMessageStatus()) {
// Success
case PUT_OK:
case FLUSH_DISK_TIMEOUT:
case FLUSH_SLAVE_TIMEOUT:
case SLAVE_NOT_AVAILABLE:
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
break;
// Failed
case CREATE_MAPEDFILE_FAILED:
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("Create mapped file failed.");
break;
case MESSAGE_ILLEGAL:
case PROPERTIES_SIZE_EXCEEDED:
response.setCode(ResponseCode.MESSAGE_ILLEGAL);
response.setRemark("The message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k.");
break;
case SERVICE_NOT_AVAILABLE:
response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);
response.setRemark("Service not available now.");
break;
case OS_PAGECACHE_BUSY:
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("OS page cache busy, please try another machine");
break;
case UNKNOWN_ERROR:
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("UNKNOWN_ERROR");
break;
default:
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("UNKNOWN_ERROR DEFAULT");
break;
}
return response;
} else {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("store putMessage return null");
}
return response;
}
} }
...@@ -19,13 +19,13 @@ package org.apache.rocketmq.broker.processor; ...@@ -19,13 +19,13 @@ package org.apache.rocketmq.broker.processor;
import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelHandlerContext;
import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ForwardRequestProcessor implements NettyRequestProcessor { public class ForwardRequestProcessor implements NettyRequestProcessor {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController; private final BrokerController brokerController;
......
...@@ -41,6 +41,8 @@ import org.apache.rocketmq.common.constant.PermName; ...@@ -41,6 +41,8 @@ import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.ExpressionType;
import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.filter.FilterAPI;
import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.ResponseCode;
...@@ -63,11 +65,9 @@ import org.apache.rocketmq.store.MessageFilter; ...@@ -63,11 +65,9 @@ import org.apache.rocketmq.store.MessageFilter;
import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.stats.BrokerStatsManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PullMessageProcessor implements NettyRequestProcessor { public class PullMessageProcessor implements NettyRequestProcessor {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController; private final BrokerController brokerController;
private List<ConsumeMessageHook> consumeMessageHookList; private List<ConsumeMessageHook> consumeMessageHookList;
......
...@@ -25,6 +25,8 @@ import org.apache.rocketmq.broker.pagecache.OneMessageTransfer; ...@@ -25,6 +25,8 @@ import org.apache.rocketmq.broker.pagecache.OneMessageTransfer;
import org.apache.rocketmq.broker.pagecache.QueryMessageTransfer; import org.apache.rocketmq.broker.pagecache.QueryMessageTransfer;
import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader;
...@@ -35,11 +37,9 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; ...@@ -35,11 +37,9 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.store.QueryMessageResult; import org.apache.rocketmq.store.QueryMessageResult;
import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class QueryMessageProcessor implements NettyRequestProcessor { public class QueryMessageProcessor implements NettyRequestProcessor {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController; private final BrokerController brokerController;
......
...@@ -17,8 +17,6 @@ ...@@ -17,8 +17,6 @@
package org.apache.rocketmq.broker.processor; package org.apache.rocketmq.broker.processor;
import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelHandlerContext;
import java.net.SocketAddress;
import java.util.List;
import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
...@@ -51,6 +49,10 @@ import org.apache.rocketmq.store.PutMessageResult; ...@@ -51,6 +49,10 @@ import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.config.StorePathConfigHelper;
import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.apache.rocketmq.store.stats.BrokerStatsManager;
import java.net.SocketAddress;
import java.util.List;
import java.util.Map;
public class SendMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor { public class SendMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor {
private List<ConsumeMessageHook> consumeMessageHookList; private List<ConsumeMessageHook> consumeMessageHookList;
...@@ -61,7 +63,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement ...@@ -61,7 +63,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
@Override @Override
public RemotingCommand processRequest(ChannelHandlerContext ctx, public RemotingCommand processRequest(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException { RemotingCommand request) throws RemotingCommandException {
SendMessageContext mqtraceContext; SendMessageContext mqtraceContext;
switch (request.getCode()) { switch (request.getCode()) {
case RequestCode.CONSUMER_SEND_MSG_BACK: case RequestCode.CONSUMER_SEND_MSG_BACK:
...@@ -97,7 +99,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement ...@@ -97,7 +99,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
throws RemotingCommandException { throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null); final RemotingCommand response = RemotingCommand.createResponseCommand(null);
final ConsumerSendMsgBackRequestHeader requestHeader = final ConsumerSendMsgBackRequestHeader requestHeader =
(ConsumerSendMsgBackRequestHeader) request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class); (ConsumerSendMsgBackRequestHeader)request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class);
if (this.hasConsumeMessageHook() && !UtilAll.isBlank(requestHeader.getOriginMsgId())) { if (this.hasConsumeMessageHook() && !UtilAll.isBlank(requestHeader.getOriginMsgId())) {
...@@ -247,8 +249,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement ...@@ -247,8 +249,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
} }
private boolean handleRetryAndDLQ(SendMessageRequestHeader requestHeader, RemotingCommand response, private boolean handleRetryAndDLQ(SendMessageRequestHeader requestHeader, RemotingCommand response,
RemotingCommand request, RemotingCommand request,
MessageExt msg, TopicConfig topicConfig) { MessageExt msg, TopicConfig topicConfig) {
String newTopic = requestHeader.getTopic(); String newTopic = requestHeader.getTopic();
if (null != newTopic && newTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { if (null != newTopic && newTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
String groupName = newTopic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length()); String groupName = newTopic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length());
...@@ -291,12 +293,12 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement ...@@ -291,12 +293,12 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
} }
private RemotingCommand sendMessage(final ChannelHandlerContext ctx, private RemotingCommand sendMessage(final ChannelHandlerContext ctx,
final RemotingCommand request, final RemotingCommand request,
final SendMessageContext sendMessageContext, final SendMessageContext sendMessageContext,
final SendMessageRequestHeader requestHeader) throws RemotingCommandException { final SendMessageRequestHeader requestHeader) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader(); final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader();
response.setOpaque(request.getOpaque()); response.setOpaque(request.getOpaque());
...@@ -343,27 +345,30 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement ...@@ -343,27 +345,30 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
msgInner.setBornHost(ctx.channel().remoteAddress()); msgInner.setBornHost(ctx.channel().remoteAddress());
msgInner.setStoreHost(this.getStoreHost()); msgInner.setStoreHost(this.getStoreHost());
msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
PutMessageResult putMessageResult = null;
if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) { Map<String, String> oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
String traFlag = msgInner.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); String traFlag = oriProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);
if (traFlag != null) { if (traFlag != null && Boolean.parseBoolean(traFlag)) {
if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) {
response.setCode(ResponseCode.NO_PERMISSION); response.setCode(ResponseCode.NO_PERMISSION);
response.setRemark( response.setRemark(
"the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] sending transaction message is forbidden"); "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1()
+ "] sending transaction message is forbidden");
return response; return response;
} }
putMessageResult = this.brokerController.getTransactionalMessageService().prepareMessage(msgInner);
} else {
putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
} }
PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt); return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt);
} }
private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand response, private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand response,
RemotingCommand request, MessageExt msg, RemotingCommand request, MessageExt msg,
SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx, SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx,
int queueIdInt) { int queueIdInt) {
if (putMessageResult == null) { if (putMessageResult == null) {
response.setCode(ResponseCode.SYSTEM_ERROR); response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("store putMessage return null"); response.setRemark("store putMessage return null");
...@@ -443,7 +448,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement ...@@ -443,7 +448,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount(); int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount();
int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes(); int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes();
int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount; int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount;
sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS); sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS);
sendMessageContext.setCommercialSendTimes(incValue); sendMessageContext.setCommercialSendTimes(incValue);
...@@ -454,7 +459,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement ...@@ -454,7 +459,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
} else { } else {
if (hasSendMessageHook()) { if (hasSendMessageHook()) {
int wroteSize = request.getBody().length; int wroteSize = request.getBody().length;
int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT);
sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE); sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE);
sendMessageContext.setCommercialSendTimes(incValue); sendMessageContext.setCommercialSendTimes(incValue);
...@@ -466,12 +471,12 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement ...@@ -466,12 +471,12 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
} }
private RemotingCommand sendBatchMessage(final ChannelHandlerContext ctx, private RemotingCommand sendBatchMessage(final ChannelHandlerContext ctx,
final RemotingCommand request, final RemotingCommand request,
final SendMessageContext sendMessageContext, final SendMessageContext sendMessageContext,
final SendMessageRequestHeader requestHeader) throws RemotingCommandException { final SendMessageRequestHeader requestHeader) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader(); final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader();
response.setOpaque(request.getOpaque()); response.setOpaque(request.getOpaque());
......
...@@ -21,15 +21,15 @@ import org.apache.rocketmq.broker.BrokerController; ...@@ -21,15 +21,15 @@ import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper; import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.config.StorePathConfigHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SlaveSynchronize { public class SlaveSynchronize {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController; private final BrokerController brokerController;
private volatile String masterAddr = null; private volatile String masterAddr = null;
......
...@@ -26,13 +26,13 @@ import org.apache.rocketmq.common.ConfigManager; ...@@ -26,13 +26,13 @@ import org.apache.rocketmq.common.ConfigManager;
import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SubscriptionGroupManager extends ConfigManager { public class SubscriptionGroupManager extends ConfigManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable = private final ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable =
new ConcurrentHashMap<String, SubscriptionGroupConfig>(1024); new ConcurrentHashMap<String, SubscriptionGroupConfig>(1024);
......
...@@ -34,14 +34,14 @@ import org.apache.rocketmq.common.MixAll; ...@@ -34,14 +34,14 @@ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.common.sysflag.TopicSysFlag; import org.apache.rocketmq.common.sysflag.TopicSysFlag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TopicConfigManager extends ConfigManager { public class TopicConfigManager extends ConfigManager {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final long LOCK_TIMEOUT_MILLIS = 3000; private static final long LOCK_TIMEOUT_MILLIS = 3000;
private transient final Lock lockTopicConfigTable = new ReentrantLock(); private transient final Lock lockTopicConfigTable = new ReentrantLock();
...@@ -66,9 +66,9 @@ public class TopicConfigManager extends ConfigManager { ...@@ -66,9 +66,9 @@ public class TopicConfigManager extends ConfigManager {
this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
} }
{ {
// MixAll.DEFAULT_TOPIC // MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC
if (this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) { if (this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) {
String topic = MixAll.DEFAULT_TOPIC; String topic = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC;
TopicConfig topicConfig = new TopicConfig(topic); TopicConfig topicConfig = new TopicConfig(topic);
this.systemTopicList.add(topic); this.systemTopicList.add(topic);
topicConfig.setReadQueueNums(this.brokerController.getBrokerConfig() topicConfig.setReadQueueNums(this.brokerController.getBrokerConfig()
...@@ -135,7 +135,7 @@ public class TopicConfigManager extends ConfigManager { ...@@ -135,7 +135,7 @@ public class TopicConfigManager extends ConfigManager {
} }
public boolean isTopicCanSendMessage(final String topic) { public boolean isTopicCanSendMessage(final String topic) {
return !topic.equals(MixAll.DEFAULT_TOPIC); return !topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC);
} }
public TopicConfig selectTopicConfig(final String topic) { public TopicConfig selectTopicConfig(final String topic) {
...@@ -156,7 +156,7 @@ public class TopicConfigManager extends ConfigManager { ...@@ -156,7 +156,7 @@ public class TopicConfigManager extends ConfigManager {
TopicConfig defaultTopicConfig = this.topicConfigTable.get(defaultTopic); TopicConfig defaultTopicConfig = this.topicConfigTable.get(defaultTopic);
if (defaultTopicConfig != null) { if (defaultTopicConfig != null) {
if (defaultTopic.equals(MixAll.DEFAULT_TOPIC)) { if (defaultTopic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) {
if (!this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) { if (!this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) {
defaultTopicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE); defaultTopicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE);
} }
...@@ -210,7 +210,7 @@ public class TopicConfigManager extends ConfigManager { ...@@ -210,7 +210,7 @@ public class TopicConfigManager extends ConfigManager {
} }
if (createNew) { if (createNew) {
this.brokerController.registerBrokerAll(false, true); this.brokerController.registerBrokerAll(false, true,true);
} }
return topicConfig; return topicConfig;
...@@ -254,7 +254,7 @@ public class TopicConfigManager extends ConfigManager { ...@@ -254,7 +254,7 @@ public class TopicConfigManager extends ConfigManager {
} }
if (createNew) { if (createNew) {
this.brokerController.registerBrokerAll(false, true); this.brokerController.registerBrokerAll(false, true,true);
} }
return topicConfig; return topicConfig;
...@@ -279,7 +279,7 @@ public class TopicConfigManager extends ConfigManager { ...@@ -279,7 +279,7 @@ public class TopicConfigManager extends ConfigManager {
this.dataVersion.nextVersion(); this.dataVersion.nextVersion();
this.persist(); this.persist();
this.brokerController.registerBrokerAll(false, true); this.brokerController.registerBrokerAll(false, true,true);
} }
} }
...@@ -299,7 +299,7 @@ public class TopicConfigManager extends ConfigManager { ...@@ -299,7 +299,7 @@ public class TopicConfigManager extends ConfigManager {
this.dataVersion.nextVersion(); this.dataVersion.nextVersion();
this.persist(); this.persist();
this.brokerController.registerBrokerAll(false, true); this.brokerController.registerBrokerAll(false, true,true);
} }
} }
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.transaction;
import io.netty.channel.Channel;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public abstract class AbstractTransactionalMessageCheckListener {
private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
private BrokerController brokerController;
private static ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(2000), new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r);
thread.setName("Transaction-msg-check-thread");
return thread;
}
});
public AbstractTransactionalMessageCheckListener() {
}
public AbstractTransactionalMessageCheckListener(BrokerController brokerController) {
this.brokerController = brokerController;
}
public void sendCheckMessage(MessageExt msgExt) throws Exception {
CheckTransactionStateRequestHeader checkTransactionStateRequestHeader = new CheckTransactionStateRequestHeader();
checkTransactionStateRequestHeader.setCommitLogOffset(msgExt.getCommitLogOffset());
checkTransactionStateRequestHeader.setOffsetMsgId(msgExt.getMsgId());
checkTransactionStateRequestHeader.setMsgId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
checkTransactionStateRequestHeader.setTransactionId(checkTransactionStateRequestHeader.getMsgId());
checkTransactionStateRequestHeader.setTranStateTableOffset(msgExt.getQueueOffset());
msgExt.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC));
msgExt.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID)));
msgExt.setStoreSize(0);
String groupId = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);
Channel channel = brokerController.getProducerManager().getAvaliableChannel(groupId);
if (channel != null) {
brokerController.getBroker2Client().checkProducerTransactionState(groupId, channel, checkTransactionStateRequestHeader, msgExt);
} else {
LOGGER.warn("Check transaction failed, channel is null. groupId={}", groupId);
}
}
public void resolveHalfMsg(final MessageExt msgExt) {
executorService.execute(new Runnable() {
@Override
public void run() {
try {
sendCheckMessage(msgExt);
} catch (Exception e) {
LOGGER.error("Send check message error!", e);
}
}
});
}
public BrokerController getBrokerController() {
return brokerController;
}
public void shutDown() {
executorService.shutdown();
}
/**
* Inject brokerController for this listener
*
* @param brokerController
*/
public void setBrokerController(BrokerController brokerController) {
this.brokerController = brokerController;
}
/**
* In order to avoid check back unlimited, we will discard the message that have been checked more than a certain
* number of times.
*
* @param msgExt Message to be discarded.
*/
public abstract void resolveDiscardMsg(MessageExt msgExt);
}
...@@ -14,37 +14,38 @@ ...@@ -14,37 +14,38 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.rocketmq.broker.transaction;
package org.apache.rocketmq.filtersrv.filter; import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.filter.MessageFilter; public class OperationResult {
private MessageExt prepareMessage;
public class FilterClassInfo { private int responseCode;
private String className;
private int classCRC;
private MessageFilter messageFilter;
public int getClassCRC() { private String responseRemark;
return classCRC;
public MessageExt getPrepareMessage() {
return prepareMessage;
} }
public void setClassCRC(int classCRC) { public void setPrepareMessage(MessageExt prepareMessage) {
this.classCRC = classCRC; this.prepareMessage = prepareMessage;
} }
public MessageFilter getMessageFilter() { public int getResponseCode() {
return messageFilter; return responseCode;
} }
public void setMessageFilter(MessageFilter messageFilter) { public void setResponseCode(int responseCode) {
this.messageFilter = messageFilter; this.responseCode = responseCode;
} }
public String getClassName() { public String getResponseRemark() {
return className; return responseRemark;
} }
public void setClassName(String className) { public void setResponseRemark(String responseRemark) {
this.className = className; this.responseRemark = responseRemark;
} }
} }
...@@ -14,9 +14,12 @@ ...@@ -14,9 +14,12 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.rocketmq.broker.transaction; package org.apache.rocketmq.broker.transaction;
/**
* This class will be removed in the version 4.4.0 and {@link OperationResult} class is recommended.
*/
@Deprecated
public class TransactionRecord { public class TransactionRecord {
// Commit Log Offset // Commit Log Offset
private long offset; private long offset;
......
...@@ -19,6 +19,10 @@ package org.apache.rocketmq.broker.transaction; ...@@ -19,6 +19,10 @@ package org.apache.rocketmq.broker.transaction;
import java.util.List; import java.util.List;
/**
* This class will be removed in ther version 4.4.0, and {@link TransactionalMessageService} class is recommended.
*/
@Deprecated
public interface TransactionStore { public interface TransactionStore {
boolean open(); boolean open();
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.transaction;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.ServiceThread;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import java.util.concurrent.atomic.AtomicBoolean;
public class TransactionalMessageCheckService extends ServiceThread {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
private BrokerController brokerController;
private final AtomicBoolean started = new AtomicBoolean(false);
public TransactionalMessageCheckService(BrokerController brokerController) {
this.brokerController = brokerController;
}
@Override
public void start() {
if (started.compareAndSet(false, true)) {
super.start();
this.brokerController.getTransactionalMessageService().open();
}
}
@Override
public void shutdown(boolean interrupt) {
if (started.compareAndSet(true, false)) {
super.shutdown(interrupt);
this.brokerController.getTransactionalMessageService().close();
this.brokerController.getTransactionalMessageCheckListener().shutDown();
}
}
@Override
public String getServiceName() {
return TransactionalMessageCheckService.class.getSimpleName();
}
@Override
public void run() {
log.info("Start transaction check service thread!");
long checkInterval = brokerController.getBrokerConfig().getTransactionCheckInterval();
while (!this.isStopped()) {
this.waitForRunning(checkInterval);
}
log.info("End transaction check service thread!");
}
@Override
protected void onWaitEnd() {
long timeout = brokerController.getBrokerConfig().getTransactionTimeOut();
int checkMax = brokerController.getBrokerConfig().getTransactionCheckMax();
long begin = System.currentTimeMillis();
log.info("Begin to check prepare message, begin time:{}", begin);
this.brokerController.getTransactionalMessageService().check(timeout, checkMax, this.brokerController.getTransactionalMessageCheckListener());
log.info("End to check prepare message, consumed time:{}", System.currentTimeMillis() - begin);
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.transaction;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.PutMessageResult;
public interface TransactionalMessageService {
/**
* Process prepare message, in common, we should put this message to storage service.
*
* @param messageInner Prepare(Half) message.
* @return Prepare message storage result.
*/
PutMessageResult prepareMessage(MessageExtBrokerInner messageInner);
/**
* Delete prepare message when this message has been committed or rolled back.
*
* @param messageExt
*/
boolean deletePrepareMessage(MessageExt messageExt);
/**
* Invoked to process commit prepare message.
*
* @param requestHeader Commit message request header.
* @return Operate result contains prepare message and relative error code.
*/
OperationResult commitMessage(EndTransactionRequestHeader requestHeader);
/**
* Invoked to roll back prepare message.
*
* @param requestHeader Prepare message request header.
* @return Operate result contains prepare message and relative error code.
*/
OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader);
/**
* Traverse uncommitted/unroll back half message and send check back request to producer to obtain transaction
* status.
*
* @param transactionTimeout The minimum time of the transactional message to be checked firstly, one message only
* exceed this time interval that can be checked.
* @param transactionCheckMax The maximum number of times the message was checked, if exceed this value, this
* message will be discarded.
* @param listener When the message is considered to be checked or discarded, the relative method of this class will
* be invoked.
*/
void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener);
/**
* Open transaction service.
*
* @return If open success, return true.
*/
boolean open();
/**
* Close transaction service.
*/
void close();
}
...@@ -31,11 +31,11 @@ import org.apache.rocketmq.broker.transaction.TransactionRecord; ...@@ -31,11 +31,11 @@ import org.apache.rocketmq.broker.transaction.TransactionRecord;
import org.apache.rocketmq.broker.transaction.TransactionStore; import org.apache.rocketmq.broker.transaction.TransactionStore;
import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.slf4j.Logger; import org.apache.rocketmq.logging.InternalLogger;
import org.slf4j.LoggerFactory; import org.apache.rocketmq.logging.InternalLoggerFactory;
public class JDBCTransactionStore implements TransactionStore { public class JDBCTransactionStore implements TransactionStore {
private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
private final JDBCTransactionStoreConfig jdbcTransactionStoreConfig; private final JDBCTransactionStoreConfig jdbcTransactionStoreConfig;
private Connection connection; private Connection connection;
private AtomicLong totalRecordsValue = new AtomicLong(0); private AtomicLong totalRecordsValue = new AtomicLong(0);
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.transaction.queue;
import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
public class DefaultTransactionalMessageCheckListener extends AbstractTransactionalMessageCheckListener {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
public DefaultTransactionalMessageCheckListener() {
super();
}
@Override
public void resolveDiscardMsg(MessageExt msgExt) {
log.error("MsgExt:{} has been checked too many times, so discard it", msgExt);
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.transaction.queue;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.common.message.MessageExt;
public class GetResult {
private MessageExt msg;
private PullResult pullResult;
public MessageExt getMsg() {
return msg;
}
public void setMsg(MessageExt msg) {
this.msg = msg;
}
public PullResult getPullResult() {
return pullResult;
}
public void setPullResult(PullResult pullResult) {
this.pullResult = pullResult;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.transaction.queue;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.client.consumer.PullStatus;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageClientIDSetter;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.logging.InnerLoggerFactory;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
public class TransactionalMessageBridge {
private static final InternalLogger LOGGER = InnerLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
private final ConcurrentHashMap<MessageQueue, MessageQueue> opQueueMap = new ConcurrentHashMap<>();
private final BrokerController brokerController;
private final MessageStore store;
private final SocketAddress storeHost;
public TransactionalMessageBridge(BrokerController brokerController, MessageStore store) {
try {
this.brokerController = brokerController;
this.store = store;
this.storeHost =
new InetSocketAddress(brokerController.getBrokerConfig().getBrokerIP1(),
brokerController.getNettyServerConfig().getListenPort());
} catch (Exception e) {
LOGGER.error("Init TransactionBridge error", e);
throw new RuntimeException(e);
}
}
public long fetchConsumeOffset(MessageQueue mq) {
long offset = brokerController.getConsumerOffsetManager().queryOffset(TransactionalMessageUtil.buildConsumerGroup(),
mq.getTopic(), mq.getQueueId());
if (offset == -1) {
offset = store.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId());
}
return offset;
}
public Set<MessageQueue> fetchMessageQueues(String topic) {
Set<MessageQueue> mqSet = new HashSet<>();
TopicConfig topicConfig = selectTopicConfig(topic);
if (topicConfig != null && topicConfig.getReadQueueNums() > 0) {
for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
MessageQueue mq = new MessageQueue();
mq.setTopic(topic);
mq.setBrokerName(brokerController.getBrokerConfig().getBrokerName());
mq.setQueueId(i);
mqSet.add(mq);
}
}
return mqSet;
}
public void updateConsumeOffset(MessageQueue mq, long offset) {
this.brokerController.getConsumerOffsetManager().commitOffset(
RemotingHelper.parseSocketAddressAddr(this.storeHost), TransactionalMessageUtil.buildConsumerGroup(), mq.getTopic(),
mq.getQueueId(), offset);
}
public PullResult getHalfMessage(int queueId, long offset, int nums) {
String group = TransactionalMessageUtil.buildConsumerGroup();
String topic = TransactionalMessageUtil.buildHalfTopic();
SubscriptionData sub = new SubscriptionData(topic, "*");
return getMessage(group, topic, queueId, offset, nums, sub);
}
public PullResult getOpMessage(int queueId, long offset, int nums) {
String group = TransactionalMessageUtil.buildConsumerGroup();
String topic = TransactionalMessageUtil.buildOpTopic();
SubscriptionData sub = new SubscriptionData(topic, "*");
return getMessage(group, topic, queueId, offset, nums, sub);
}
private PullResult getMessage(String group, String topic, int queueId, long offset, int nums,
SubscriptionData sub) {
GetMessageResult getMessageResult = store.getMessage(group, topic, queueId, offset, nums, null);
if (getMessageResult != null) {
PullStatus pullStatus = PullStatus.NO_NEW_MSG;
List<MessageExt> foundList = null;
switch (getMessageResult.getStatus()) {
case FOUND:
pullStatus = PullStatus.FOUND;
foundList = decodeMsgList(getMessageResult);
this.brokerController.getBrokerStatsManager().incGroupGetNums(group, topic,
getMessageResult.getMessageCount());
this.brokerController.getBrokerStatsManager().incGroupGetSize(group, topic,
getMessageResult.getBufferTotalSize());
this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount());
this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId,
this.brokerController.getMessageStore().now() - foundList.get(foundList.size() - 1)
.getStoreTimestamp());
break;
case NO_MATCHED_MESSAGE:
pullStatus = PullStatus.NO_MATCHED_MSG;
LOGGER.warn("No matched message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}",
getMessageResult.getStatus(), topic, group, offset);
break;
case NO_MESSAGE_IN_QUEUE:
pullStatus = PullStatus.NO_NEW_MSG;
LOGGER.warn("No new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}",
getMessageResult.getStatus(), topic, group, offset);
break;
case MESSAGE_WAS_REMOVING:
case NO_MATCHED_LOGIC_QUEUE:
case OFFSET_FOUND_NULL:
case OFFSET_OVERFLOW_BADLY:
case OFFSET_OVERFLOW_ONE:
case OFFSET_TOO_SMALL:
pullStatus = PullStatus.OFFSET_ILLEGAL;
LOGGER.warn("Offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}",
getMessageResult.getStatus(), topic, group, offset);
break;
default:
assert false;
break;
}
return new PullResult(pullStatus, getMessageResult.getNextBeginOffset(), getMessageResult.getMinOffset(),
getMessageResult.getMaxOffset(), foundList);
} else {
LOGGER.error("Get message from store return null. topic={}, groupId={}, requestOffset={}", topic, group,
offset);
return null;
}
}
private List<MessageExt> decodeMsgList(GetMessageResult getMessageResult) {
List<MessageExt> foundList = new ArrayList<>();
try {
List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();
for (ByteBuffer bb : messageBufferList) {
MessageExt msgExt = MessageDecoder.decode(bb);
foundList.add(msgExt);
}
} finally {
getMessageResult.release();
}
return foundList;
}
public PutMessageResult putHalfMessage(MessageExtBrokerInner messageInner) {
return store.putMessage(parseHalfMessageInner(messageInner));
}
private MessageExtBrokerInner parseHalfMessageInner(MessageExtBrokerInner msgInner) {
MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC, msgInner.getTopic());
MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID,
String.valueOf(msgInner.getQueueId()));
msgInner.setSysFlag(
MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), MessageSysFlag.TRANSACTION_NOT_TYPE));
msgInner.setTopic(TransactionalMessageUtil.buildHalfTopic());
msgInner.setQueueId(0);
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
return msgInner;
}
public boolean putOpMessage(MessageExt messageExt, String opType) {
MessageQueue messageQueue = new MessageQueue(messageExt.getTopic(),
this.brokerController.getBrokerConfig().getBrokerName(), messageExt.getQueueId());
if (TransactionalMessageUtil.REMOVETAG.equals(opType)) {
return addRemoveTagInTransactionOp(messageExt, messageQueue);
}
return true;
}
public PutMessageResult putMessageReturnResult(MessageExtBrokerInner messageInner) {
LOGGER.debug("[BUG-TO-FIX] Thread:{} msgID:{}", Thread.currentThread().getName(), messageInner.getMsgId());
return store.putMessage(messageInner);
}
public boolean putMessage(MessageExtBrokerInner messageInner) {
PutMessageResult putMessageResult = store.putMessage(messageInner);
if (putMessageResult != null
&& putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) {
return true;
} else {
LOGGER.error("Put message failed, topic: {}, queueId: {}, msgId: {}",
messageInner.getTopic(), messageInner.getQueueId(), messageInner.getMsgId());
return false;
}
}
public MessageExtBrokerInner renewImmunityHalfMessageInner(MessageExt msgExt) {
MessageExtBrokerInner msgInner = renewHalfMessageInner(msgExt);
String queueOffsetFromPrepare = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET);
if (null != queueOffsetFromPrepare) {
MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET,
String.valueOf(queueOffsetFromPrepare));
} else {
MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET,
String.valueOf(msgExt.getQueueOffset()));
}
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
return msgInner;
}
public MessageExtBrokerInner renewHalfMessageInner(MessageExt msgExt) {
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
msgInner.setTopic(msgExt.getTopic());
msgInner.setBody(msgExt.getBody());
msgInner.setQueueId(msgExt.getQueueId());
msgInner.setMsgId(msgExt.getMsgId());
msgInner.setSysFlag(msgExt.getSysFlag());
msgInner.setTags(msgExt.getTags());
msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags()));
MessageAccessor.setProperties(msgInner, msgExt.getProperties());
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));
msgInner.setBornTimestamp(msgExt.getBornTimestamp());
msgInner.setBornHost(msgExt.getBornHost());
msgInner.setStoreHost(msgExt.getStoreHost());
msgInner.setWaitStoreMsgOK(false);
return msgInner;
}
private MessageExtBrokerInner makeOpMessageInner(Message message, MessageQueue messageQueue) {
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
msgInner.setTopic(message.getTopic());
msgInner.setBody(message.getBody());
msgInner.setQueueId(messageQueue.getQueueId());
msgInner.setTags(message.getTags());
msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags()));
msgInner.setSysFlag(0);
MessageAccessor.setProperties(msgInner, message.getProperties());
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(message.getProperties()));
msgInner.setBornTimestamp(System.currentTimeMillis());
msgInner.setBornHost(this.storeHost);
msgInner.setStoreHost(this.storeHost);
msgInner.setWaitStoreMsgOK(false);
MessageClientIDSetter.setUniqID(msgInner);
return msgInner;
}
private TopicConfig selectTopicConfig(String topic) {
TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic);
if (topicConfig == null) {
topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(
topic, 1, PermName.PERM_WRITE | PermName.PERM_READ, 0);
}
return topicConfig;
}
/**
* Use this function while transaction msg is committed or rollback write a flag 'd' to operation queue for the
* msg's offset
*
* @param messageExt Op message
* @param messageQueue Op message queue
* @return This method will always return true.
*/
private boolean addRemoveTagInTransactionOp(MessageExt messageExt, MessageQueue messageQueue) {
Message message = new Message(TransactionalMessageUtil.buildOpTopic(), TransactionalMessageUtil.REMOVETAG,
String.valueOf(messageExt.getQueueOffset()).getBytes(TransactionalMessageUtil.charset));
writeOp(message, messageQueue);
return true;
}
private void writeOp(Message message, MessageQueue mq) {
MessageQueue opQueue;
if (opQueueMap.containsKey(mq)) {
opQueue = opQueueMap.get(mq);
} else {
opQueue = getOpQueueByHalf(mq);
MessageQueue oldQueue = opQueueMap.putIfAbsent(mq, opQueue);
if (oldQueue != null) {
opQueue = oldQueue;
}
}
if (opQueue == null) {
opQueue = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), mq.getBrokerName(), mq.getQueueId());
}
putMessage(makeOpMessageInner(message, opQueue));
}
private MessageQueue getOpQueueByHalf(MessageQueue halfMQ) {
MessageQueue opQueue = new MessageQueue();
opQueue.setTopic(TransactionalMessageUtil.buildOpTopic());
opQueue.setBrokerName(halfMQ.getBrokerName());
opQueue.setQueueId(halfMQ.getQueueId());
return opQueue;
}
public MessageExt lookMessageByOffset(final long commitLogOffset) {
return this.store.lookMessageByOffset(commitLogOffset);
}
public BrokerController getBrokerController() {
return brokerController;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.transaction.queue;
import org.apache.rocketmq.common.MixAll;
import java.nio.charset.Charset;
public class TransactionalMessageUtil {
public static final String REMOVETAG = "d";
public static Charset charset = Charset.forName("utf-8");
public static String buildOpTopic() {
return MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC;
}
public static String buildHalfTopic() {
return MixAll.RMQ_SYS_TRANS_HALF_TOPIC;
}
public static String buildConsumerGroup() {
return MixAll.CID_SYS_RMQ_TRANS;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.util;
import java.util.concurrent.atomic.AtomicInteger;
public class PositiveAtomicCounter {
private static final int MASK = 0x7FFFFFFF;
private final AtomicInteger atom;
public PositiveAtomicCounter() {
atom = new AtomicInteger(0);
}
public final int incrementAndGet() {
final int rt = atom.incrementAndGet();
return rt & MASK;
}
public int intValue() {
return atom.intValue();
}
}
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to
* You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package org.apache.rocketmq.broker.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
public class ServiceProvider {
private final static Logger LOG = LoggerFactory
.getLogger(ServiceProvider.class);
/**
* A reference to the classloader that loaded this class. It's more efficient to compute it once and cache it here.
*/
private static ClassLoader thisClassLoader;
/**
* JDK1.3+ <a href= "http://java.sun.com/j2se/1.3/docs/guide/jar/jar.html#Service%20Provider" > 'Service Provider' specification</a>.
*/
public static final String TRANSACTION_SERVICE_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService";
public static final String TRANSACTION_LISTENER_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener";
static {
thisClassLoader = getClassLoader(ServiceProvider.class);
}
/**
* Returns a string that uniquely identifies the specified object, including its class.
* <p>
* The returned string is of form "classname@hashcode", ie is the same as the return value of the Object.toString() method, but works even when the specified object's class has overidden the toString method.
*
* @param o may be null.
* @return a string of form classname@hashcode, or "null" if param o is null.
*/
protected static String objectId(Object o) {
if (o == null) {
return "null";
} else {
return o.getClass().getName() + "@" + System.identityHashCode(o);
}
}
protected static ClassLoader getClassLoader(Class<?> clazz) {
try {
return clazz.getClassLoader();
} catch (SecurityException e) {
LOG.error("Unable to get classloader for class {} due to security restrictions !",
clazz, e.getMessage());
throw e;
}
}
protected static ClassLoader getContextClassLoader() {
ClassLoader classLoader = null;
try {
classLoader = Thread.currentThread().getContextClassLoader();
} catch (SecurityException ex) {
/**
* The getContextClassLoader() method throws SecurityException when the context
* class loader isn't an ancestor of the calling class's class
* loader, or if security permissions are restricted.
*/
}
return classLoader;
}
protected static InputStream getResourceAsStream(ClassLoader loader, String name) {
if (loader != null) {
return loader.getResourceAsStream(name);
} else {
return ClassLoader.getSystemResourceAsStream(name);
}
}
public static <T> List<T> load(String name, Class<?> clazz) {
LOG.info("Looking for a resource file of name [{}] ...", name);
List<T> services = new ArrayList<T>();
try {
ArrayList<String> names = new ArrayList<String>();
final InputStream is = getResourceAsStream(getContextClassLoader(), name);
if (is != null) {
BufferedReader reader;
try {
reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
} catch (java.io.UnsupportedEncodingException e) {
reader = new BufferedReader(new InputStreamReader(is));
}
String serviceName = reader.readLine();
while (serviceName != null && !"".equals(serviceName)) {
LOG.info(
"Creating an instance as specified by file {} which was present in the path of the context classloader.",
name);
if (!names.contains(serviceName)) {
names.add(serviceName);
}
services.add((T)initService(getContextClassLoader(), serviceName, clazz));
serviceName = reader.readLine();
}
reader.close();
} else {
// is == null
LOG.warn("No resource file with name [{}] found.", name);
}
} catch (Exception e) {
LOG.error("Error occured when looking for resource file " + name, e);
}
return services;
}
public static <T> T loadClass(String name, Class<?> clazz) {
final InputStream is = getResourceAsStream(getContextClassLoader(), name);
BufferedReader reader;
try {
try {
reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
} catch (java.io.UnsupportedEncodingException e) {
reader = new BufferedReader(new InputStreamReader(is));
}
String serviceName = reader.readLine();
reader.close();
if (serviceName != null && !"".equals(serviceName)) {
return initService(getContextClassLoader(), serviceName, clazz);
} else {
LOG.warn("ServiceName is empty!");
return null;
}
} catch (Exception e) {
LOG.error("Error occured when looking for resource file " + name, e);
}
return null;
}
protected static <T> T initService(ClassLoader classLoader, String serviceName, Class<?> clazz) {
Class<?> serviceClazz = null;
try {
if (classLoader != null) {
try {
// Warning: must typecast here & allow exception to be generated/caught & recast properly
serviceClazz = classLoader.loadClass(serviceName);
if (clazz.isAssignableFrom(serviceClazz)) {
LOG.info("Loaded class {} from classloader {}", serviceClazz.getName(),
objectId(classLoader));
} else {
// This indicates a problem with the ClassLoader tree. An incompatible ClassLoader was used to load the implementation.
LOG.error(
"Class {} loaded from classloader {} does not extend {} as loaded by this classloader.",
new Object[] {serviceClazz.getName(),
objectId(serviceClazz.getClassLoader()), clazz.getName()});
}
return (T)serviceClazz.newInstance();
} catch (ClassNotFoundException ex) {
if (classLoader == thisClassLoader) {
// Nothing more to try, onwards.
LOG.warn("Unable to locate any class {} via classloader", serviceName,
objectId(classLoader));
throw ex;
}
// Ignore exception, continue
} catch (NoClassDefFoundError e) {
if (classLoader == thisClassLoader) {
// Nothing more to try, onwards.
LOG.warn(
"Class {} cannot be loaded via classloader {}.it depends on some other class that cannot be found.",
serviceClazz, objectId(classLoader));
throw e;
}
// Ignore exception, continue
}
}
} catch (Exception e) {
LOG.error("Unable to init service.", e);
}
return (T)serviceClazz;
}
}
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
package org.apache.rocketmq.broker; package org.apache.rocketmq.broker;
import java.io.File;
import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyClientConfig;
...@@ -25,16 +26,10 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; ...@@ -25,16 +26,10 @@ import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.junit.After; import org.junit.After;
import org.junit.Test; import org.junit.Test;
import java.io.File;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
public class BrokerControllerTest { public class BrokerControllerTest {
/**
* Tests if the controller can be properly stopped and started.
*
* @throws Exception If fails.
*/
@Test @Test
public void testBrokerRestart() throws Exception { public void testBrokerRestart() throws Exception {
BrokerController brokerController = new BrokerController( BrokerController brokerController = new BrokerController(
...@@ -48,7 +43,7 @@ public class BrokerControllerTest { ...@@ -48,7 +43,7 @@ public class BrokerControllerTest {
} }
@After @After
public void destory() { public void destroy() {
UtilAll.deleteFile(new File(new MessageStoreConfig().getStorePathRootDir())); UtilAll.deleteFile(new File(new MessageStoreConfig().getStorePathRootDir()));
} }
} }
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import io.netty.channel.ChannelHandlerContext;
import java.lang.reflect.Field;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.broker.out.BrokerOuterAPI;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.namesrv.RegisterBrokerResult;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionResponseHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class BrokerOuterAPITest {
@Mock
private ChannelHandlerContext handlerContext;
@Spy
private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
@Mock
private MessageStore messageStore;
private String clusterName = "clusterName";
private String brokerName = "brokerName";
private String brokerAddr = "brokerAddr";
private long brokerId = 0L;
private String nameserver1 = "127.0.0.1";
private String nameserver2 = "127.0.0.2";
private String nameserver3 = "127.0.0.3";
private int timeOut = 3000;
@Mock
private NettyRemotingClient nettyRemotingClient;
private BrokerOuterAPI brokerOuterAPI;
public void init() throws Exception {
brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig(), null);
Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
field.setAccessible(true);
field.set(brokerOuterAPI, nettyRemotingClient);
}
@Test
public void test_needRegister_normal() throws Exception {
init();
brokerOuterAPI.start();
final RemotingCommand response = buildResponse(Boolean.TRUE);
TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));
when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenReturn(response);
List<Boolean> booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut);
assertTrue(booleanList.size() > 0);
assertEquals(false, booleanList.contains(Boolean.FALSE));
}
@Test
public void test_needRegister_timeout() throws Exception {
init();
brokerOuterAPI.start();
TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));
when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenAnswer(new Answer<RemotingCommand>() {
@Override
public RemotingCommand answer(InvocationOnMock invocation) throws Throwable {
if (invocation.getArgument(0) == nameserver1) {
return buildResponse(Boolean.TRUE);
} else if (invocation.getArgument(0) == nameserver2) {
return buildResponse(Boolean.FALSE);
} else if (invocation.getArgument(0) == nameserver3) {
TimeUnit.MILLISECONDS.sleep(timeOut + 20);
return buildResponse(Boolean.TRUE);
}
return buildResponse(Boolean.TRUE);
}
});
List<Boolean> booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut);
assertEquals(2, booleanList.size());
boolean success = Iterables.any(booleanList,
new Predicate<Boolean>() {
public boolean apply(Boolean input) {
return input ? true : false;
}
});
assertEquals(true, success);
}
@Test
public void test_register_normal() throws Exception {
init();
brokerOuterAPI.start();
final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class);
final RegisterBrokerResponseHeader responseHeader = (RegisterBrokerResponseHeader) response.readCustomHeader();
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));
when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenReturn(response);
List<RegisterBrokerResult> registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr, brokerName, brokerId, "hasServerAddr", topicConfigSerializeWrapper, Lists.<String>newArrayList(), false, timeOut, true);
assertTrue(registerBrokerResultList.size() > 0);
}
@Test
public void test_register_timeout() throws Exception {
init();
brokerOuterAPI.start();
final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class);
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));
when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenAnswer(new Answer<RemotingCommand>() {
@Override
public RemotingCommand answer(InvocationOnMock invocation) throws Throwable {
if (invocation.getArgument(0) == nameserver1) {
return response;
} else if (invocation.getArgument(0) == nameserver2) {
return response;
} else if (invocation.getArgument(0) == nameserver3) {
TimeUnit.MILLISECONDS.sleep(timeOut + 20);
return response;
}
return response;
}
});
List<RegisterBrokerResult> registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr, brokerName, brokerId, "hasServerAddr", topicConfigSerializeWrapper, Lists.<String>newArrayList(), false, timeOut, true);
assertEquals(2, registerBrokerResultList.size());
}
private RemotingCommand buildResponse(Boolean changed) {
final RemotingCommand response = RemotingCommand.createResponseCommand(QueryDataVersionResponseHeader.class);
final QueryDataVersionResponseHeader responseHeader = (QueryDataVersionResponseHeader) response.readCustomHeader();
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
responseHeader.setChanged(changed);
return response;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.processor;
import io.netty.channel.ChannelHandlerContext;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.transaction.OperationResult;
import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.store.AppendMessageResult;
import org.apache.rocketmq.store.AppendMessageStatus;
import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class EndTransactionProcessorTest {
private EndTransactionProcessor endTransactionProcessor;
@Mock
private ChannelHandlerContext handlerContext;
@Spy
private BrokerController
brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(),
new MessageStoreConfig());
@Mock
private MessageStore messageStore;
@Mock
private TransactionalMessageService transactionMsgService;
@Before
public void init() {
brokerController.setMessageStore(messageStore);
brokerController.setTransactionalMessageService(transactionMsgService);
endTransactionProcessor = new EndTransactionProcessor(brokerController);
}
private OperationResult createResponse(int status){
OperationResult response = new OperationResult();
response.setPrepareMessage(createDefaultMessageExt());
response.setResponseCode(status);
response.setResponseRemark(null);
return response;
}
@Test
public void testProcessRequest() throws RemotingCommandException {
when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS));
when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult
(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, false);
RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
}
@Test
public void testProcessRequest_CheckMessage() throws RemotingCommandException {
when(transactionMsgService.commitMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS));
when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult
(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_COMMIT_TYPE, true);
RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
}
@Test
public void testProcessRequest_NotType() throws RemotingCommandException {
RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_NOT_TYPE, true);
RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);
assertThat(response).isNull();
}
@Test
public void testProcessRequest_RollBack() throws RemotingCommandException {
when(transactionMsgService.rollbackMessage(any(EndTransactionRequestHeader.class))).thenReturn(createResponse(ResponseCode.SUCCESS));
RemotingCommand request = createEndTransactionMsgCommand(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE, true);
RemotingCommand response = endTransactionProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
}
private MessageExt createDefaultMessageExt() {
MessageExt messageExt = new MessageExt();
messageExt.setMsgId("12345678");
messageExt.setQueueId(0);
messageExt.setCommitLogOffset(123456789L);
messageExt.setQueueOffset(1234);
MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_REAL_QUEUE_ID, "0");
MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true");
MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_PRODUCER_GROUP, "testTransactionGroup");
return messageExt;
}
private EndTransactionRequestHeader createEndTransactionRequestHeader(int status, boolean isCheckMsg) {
EndTransactionRequestHeader header = new EndTransactionRequestHeader();
header.setCommitLogOffset(123456789L);
header.setFromTransactionCheck(isCheckMsg);
header.setCommitOrRollback(status);
header.setMsgId("12345678");
header.setTransactionId("123");
header.setProducerGroup("testTransactionGroup");
header.setTranStateTableOffset(1234L);
return header;
}
private RemotingCommand createEndTransactionMsgCommand(int status, boolean isCheckMsg) {
EndTransactionRequestHeader header = createEndTransactionRequestHeader(status, isCheckMsg);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, header);
request.makeCustomHeaderToNet();
return request;
}
}
...@@ -18,19 +18,20 @@ package org.apache.rocketmq.broker.processor; ...@@ -18,19 +18,20 @@ package org.apache.rocketmq.broker.processor;
import io.netty.channel.Channel; import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelHandlerContext;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
import org.apache.rocketmq.broker.mqtrace.SendMessageHook; import org.apache.rocketmq.broker.mqtrace.SendMessageHook;
import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader;
import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig;
...@@ -51,6 +52,11 @@ import org.mockito.invocation.InvocationOnMock; ...@@ -51,6 +52,11 @@ import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner; import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer; import org.mockito.stubbing.Answer;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyLong;
...@@ -68,6 +74,9 @@ public class SendMessageProcessorTest { ...@@ -68,6 +74,9 @@ public class SendMessageProcessorTest {
@Mock @Mock
private MessageStore messageStore; private MessageStore messageStore;
@Mock
private TransactionalMessageService transactionMsgService;
private String topic = "FooBar"; private String topic = "FooBar";
private String group = "FooBarGroup"; private String group = "FooBarGroup";
...@@ -177,17 +186,57 @@ public class SendMessageProcessorTest { ...@@ -177,17 +186,57 @@ public class SendMessageProcessorTest {
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
} }
private RemotingCommand createSendMsgCommand(int requestCode) { @Test
public void testProcessRequest_Transaction() throws RemotingCommandException {
brokerController.setTransactionalMessageService(transactionMsgService);
when(brokerController.getTransactionalMessageService().prepareMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
RemotingCommand request = createSendTransactionMsgCommand(RequestCode.SEND_MESSAGE);
final RemotingCommand[] response = new RemotingCommand[1];
doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
response[0] = invocation.getArgument(0);
return null;
}
}).when(handlerContext).writeAndFlush(any(Object.class));
RemotingCommand responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);
if (responseToReturn != null) {
assertThat(response[0]).isNull();
response[0] = responseToReturn;
}
assertThat(response[0].getCode()).isEqualTo(ResponseCode.SUCCESS);
}
private RemotingCommand createSendTransactionMsgCommand(int requestCode) {
SendMessageRequestHeader header = createSendMsgRequestHeader();
int sysFlag = header.getSysFlag();
Map<String, String> oriProps = MessageDecoder.string2messageProperties(header.getProperties());
oriProps.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, "true");
header.setProperties(MessageDecoder.messageProperties2String(oriProps));
sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE;
header.setSysFlag(sysFlag);
RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, header);
request.setBody(new byte[] {'a'});
request.makeCustomHeaderToNet();
return request;
}
private SendMessageRequestHeader createSendMsgRequestHeader() {
SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
requestHeader.setProducerGroup(group); requestHeader.setProducerGroup(group);
requestHeader.setTopic(topic); requestHeader.setTopic(topic);
requestHeader.setDefaultTopic(MixAll.DEFAULT_TOPIC); requestHeader.setDefaultTopic(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC);
requestHeader.setDefaultTopicQueueNums(3); requestHeader.setDefaultTopicQueueNums(3);
requestHeader.setQueueId(1); requestHeader.setQueueId(1);
requestHeader.setSysFlag(0); requestHeader.setSysFlag(0);
requestHeader.setBornTimestamp(System.currentTimeMillis()); requestHeader.setBornTimestamp(System.currentTimeMillis());
requestHeader.setFlag(124); requestHeader.setFlag(124);
requestHeader.setReconsumeTimes(0); requestHeader.setReconsumeTimes(0);
return requestHeader;
}
private RemotingCommand createSendMsgCommand(int requestCode) {
SendMessageRequestHeader requestHeader = createSendMsgRequestHeader();
RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader); RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader);
request.setBody(new byte[] {'a'}); request.setBody(new byte[] {'a'});
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.transaction.queue;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class DefaultTransactionalMessageCheckListenerTest {
private DefaultTransactionalMessageCheckListener listener;
@Spy
private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),
new NettyClientConfig(), new MessageStoreConfig());
@Before
public void init() {
listener = new DefaultTransactionalMessageCheckListener();
listener.setBrokerController(brokerController);
}
@Test
public void testResolveHalfMsg() {
listener.resolveHalfMsg(createMessageExt());
}
@Test
public void testSendCheckMessage() throws Exception{
MessageExt messageExt = createMessageExt();
listener.sendCheckMessage(messageExt);
}
@Test
public void sendCheckMessage(){
listener.resolveDiscardMsg(createMessageExt());
}
private MessageExtBrokerInner createMessageExt() {
MessageExtBrokerInner inner = new MessageExtBrokerInner();
MessageAccessor.putProperty(inner,MessageConst.PROPERTY_REAL_QUEUE_ID,"1");
MessageAccessor.putProperty(inner,MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX,"1234255");
MessageAccessor.putProperty(inner,MessageConst.PROPERTY_REAL_TOPIC,"realTopic");
inner.setTransactionId(inner.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
inner.setBody("check".getBytes());
inner.setMsgId("12344567890");
inner.setQueueId(0);
return inner;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.transaction.queue;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.client.consumer.PullStatus;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.store.AppendMessageResult;
import org.apache.rocketmq.store.AppendMessageStatus;
import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.GetMessageStatus;
import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.MessageFilter;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.Map;
import java.util.Set;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class TransactionalMessageBridgeTest {
private TransactionalMessageBridge transactionBridge;
@Spy
private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),
new NettyClientConfig(), new MessageStoreConfig());
@Mock
private MessageStore messageStore;
@Before
public void init() {
brokerController.setMessageStore(messageStore);
transactionBridge = new TransactionalMessageBridge(brokerController, messageStore);
}
@Test
public void testPutOpMessage() {
boolean isSuccess = transactionBridge.putOpMessage(createMessageBrokerInner(), TransactionalMessageUtil.REMOVETAG);
assertThat(isSuccess).isTrue();
}
@Test
public void testPutHalfMessage() {
when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult
(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
PutMessageResult result = transactionBridge.putHalfMessage(createMessageBrokerInner());
assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);
}
@Test
public void testFetchMessageQueues() {
Set<MessageQueue> messageQueues = transactionBridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC);
assertThat(messageQueues.size()).isEqualTo(1);
}
@Test
public void testFetchConsumeOffset() {
MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(),
0);
long offset = transactionBridge.fetchConsumeOffset(mq);
assertThat(offset).isGreaterThan(-1);
}
@Test
public void updateConsumeOffset() {
MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(),
0);
transactionBridge.updateConsumeOffset(mq, 0);
}
@Test
public void testGetHalfMessage() {
when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE));
PullResult result = transactionBridge.getHalfMessage(0, 0, 1);
assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG);
}
@Test
public void testGetOpMessage() {
when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE));
PullResult result = transactionBridge.getOpMessage(0, 0, 1);
assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG);
}
@Test
public void testPutMessageReturnResult() {
when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult
(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
PutMessageResult result = transactionBridge.putMessageReturnResult(createMessageBrokerInner());
assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK);
}
@Test
public void testPutMessage() {
when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult
(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
Boolean success = transactionBridge.putMessage(createMessageBrokerInner());
assertThat(success).isEqualTo(true);
}
@Test
public void testRenewImmunityHalfMessageInner() {
MessageExt messageExt = createMessageBrokerInner();
final String offset = "123456789";
MessageExtBrokerInner msgInner = transactionBridge.renewImmunityHalfMessageInner(messageExt);
MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET,offset);
assertThat(msgInner).isNotNull();
Map<String,String> properties = msgInner.getProperties();
assertThat(properties).isNotNull();
String resOffset = properties.get(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET);
assertThat(resOffset).isEqualTo(offset);
}
@Test
public void testRenewHalfMessageInner() {
MessageExt messageExt = new MessageExt();
long bornTimeStamp = messageExt.getBornTimestamp();
MessageExt messageExtRes = transactionBridge.renewHalfMessageInner(messageExt);
assertThat( messageExtRes.getBornTimestamp()).isEqualTo(bornTimeStamp);
}
@Test
public void testLookMessageByOffset(){
when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt());
MessageExt messageExt = transactionBridge.lookMessageByOffset(123);
assertThat(messageExt).isNotNull();
}
private GetMessageResult createGetMessageResult(GetMessageStatus status) {
GetMessageResult getMessageResult = new GetMessageResult();
getMessageResult.setStatus(status);
getMessageResult.setMinOffset(100);
getMessageResult.setMaxOffset(1024);
getMessageResult.setNextBeginOffset(516);
return getMessageResult;
}
private MessageExtBrokerInner createMessageBrokerInner() {
MessageExtBrokerInner inner = new MessageExtBrokerInner();
inner.setTransactionId("12342123444");
inner.setBornTimestamp(System.currentTimeMillis());
inner.setBody("prepare".getBytes());
inner.setMsgId("123456-123");
inner.setQueueId(0);
inner.setTopic("hello");
return inner;
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.transaction.queue;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;
import org.apache.rocketmq.broker.transaction.OperationResult;
import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.client.consumer.PullStatus;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.store.AppendMessageResult;
import org.apache.rocketmq.store.AppendMessageStatus;
import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class TransactionalMessageServiceImplTest {
private TransactionalMessageService queueTransactionMsgService;
@Mock
private TransactionalMessageBridge bridge;
@Spy
private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),
new NettyClientConfig(), new MessageStoreConfig());
@Mock
private AbstractTransactionalMessageCheckListener listener;
@Before
public void init() {
listener.setBrokerController(brokerController);
queueTransactionMsgService = new TransactionalMessageServiceImpl(bridge);
brokerController.getMessageStoreConfig().setFileReservedTime(3);
}
@Test
public void testPrepareMessage() {
MessageExtBrokerInner inner = createMessageBrokerInner();
when(bridge.putHalfMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult
(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
PutMessageResult result = queueTransactionMsgService.prepareMessage(inner);
assert result.isOk();
}
@Test
public void testCommitMessage() {
when(bridge.lookMessageByOffset(anyLong())).thenReturn(createMessageBrokerInner());
OperationResult result = queueTransactionMsgService.commitMessage(createEndTransactionRequestHeader(MessageSysFlag.TRANSACTION_COMMIT_TYPE));
assertThat(result.getResponseCode()).isEqualTo(ResponseCode.SUCCESS);
}
@Test
public void testRollbackMessage() {
when(bridge.lookMessageByOffset(anyLong())).thenReturn(createMessageBrokerInner());
OperationResult result = queueTransactionMsgService.commitMessage(createEndTransactionRequestHeader(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE));
assertThat(result.getResponseCode()).isEqualTo(ResponseCode.SUCCESS);
}
@Test
public void testCheck_withDiscard() {
when(bridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(MixAll.RMQ_SYS_TRANS_HALF_TOPIC));
when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createDiscardPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 5, "hellp", 1));
when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 6, "hellp", 0));
when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createOpPulResult(MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, "10", 1));
long timeOut = this.brokerController.getBrokerConfig().getTransactionTimeOut();
int checkMax = this.brokerController.getBrokerConfig().getTransactionCheckMax();
final AtomicInteger checkMessage = new AtomicInteger(0);
doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocation) {
checkMessage.addAndGet(1);
return null;
}
}).when(listener).resolveDiscardMsg(any(MessageExt.class));
queueTransactionMsgService.check(timeOut, checkMax, listener);
assertThat(checkMessage.get()).isEqualTo(1);
}
@Test
public void testCheck_withCheck() {
when(bridge.fetchMessageQueues(MixAll.RMQ_SYS_TRANS_HALF_TOPIC)).thenReturn(createMessageQueueSet(MixAll.RMQ_SYS_TRANS_HALF_TOPIC));
when(bridge.getHalfMessage(0, 0, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 5, "hello", 1));
when(bridge.getHalfMessage(0, 1, 1)).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_HALF_TOPIC, 6, "hellp", 0));
when(bridge.getOpMessage(anyInt(), anyLong(), anyInt())).thenReturn(createPullResult(MixAll.RMQ_SYS_TRANS_OP_HALF_TOPIC, 1, "5", 0));
when(bridge.getBrokerController()).thenReturn(this.brokerController);
when(bridge.renewHalfMessageInner(any(MessageExtBrokerInner.class))).thenReturn(createMessageBrokerInner());
when(bridge.putMessageReturnResult(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult
(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
long timeOut = this.brokerController.getBrokerConfig().getTransactionTimeOut();
final int checkMax = this.brokerController.getBrokerConfig().getTransactionCheckMax();
final AtomicInteger checkMessage = new AtomicInteger(0);
doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocation) {
checkMessage.addAndGet(1);
return checkMessage;
}
}).when(listener).resolveHalfMsg(any(MessageExt.class));
queueTransactionMsgService.check(timeOut, checkMax, listener);
assertThat(checkMessage.get()).isEqualTo(1);
}
@Test
public void testDeletePrepareMessage() {
when(bridge.putOpMessage(any(MessageExt.class), anyString())).thenReturn(true);
boolean res = queueTransactionMsgService.deletePrepareMessage(createMessageBrokerInner());
assertThat(res).isTrue();
}
@Test
public void testOpen() {
boolean isOpen = queueTransactionMsgService.open();
assertThat(isOpen).isTrue();
}
private PullResult createDiscardPullResult(String topic, long queueOffset, String body, int size) {
PullResult result = createPullResult(topic, queueOffset, body, size);
List<MessageExt> msgs = result.getMsgFoundList();
for (MessageExt msg : msgs) {
msg.putUserProperty(MessageConst.PROPERTY_TRANSACTION_CHECK_TIMES, "100000");
}
return result;
}
private PullResult createPullResult(String topic, long queueOffset, String body, int size) {
PullResult result = null;
if (0 == size) {
result = new PullResult(PullStatus.NO_NEW_MSG, 1, 0, 1,
null);
} else {
result = new PullResult(PullStatus.FOUND, 1, 0, 1,
getMessageList(queueOffset, topic, body, 1));
return result;
}
return result;
}
private PullResult createOpPulResult(String topic, long queueOffset, String body, int size) {
PullResult result = createPullResult(topic, queueOffset, body, size);
List<MessageExt> msgs = result.getMsgFoundList();
for (MessageExt msg : msgs) {
msg.setTags(TransactionalMessageUtil.REMOVETAG);
}
return result;
}
private PullResult createImmunityPulResult(String topic, long queueOffset, String body, int size) {
PullResult result = createPullResult(topic, queueOffset, body, size);
List<MessageExt> msgs = result.getMsgFoundList();
for (MessageExt msg : msgs) {
msg.putUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, "0");
}
return result;
}
private List<MessageExt> getMessageList(long queueOffset, String topic, String body, int size) {
List<MessageExt> msgs = new ArrayList<>();
for (int i = 0; i < size; i++) {
MessageExt messageExt = createMessageBrokerInner(queueOffset, topic, body);
msgs.add(messageExt);
}
return msgs;
}
private Set<MessageQueue> createMessageQueueSet(String topic) {
Set<MessageQueue> messageQueues = new HashSet<>();
MessageQueue messageQueue = new MessageQueue(topic, "DefaultCluster", 0);
messageQueues.add(messageQueue);
return messageQueues;
}
private EndTransactionRequestHeader createEndTransactionRequestHeader(int status) {
EndTransactionRequestHeader header = new EndTransactionRequestHeader();
header.setCommitLogOffset(123456789L);
header.setCommitOrRollback(status);
header.setMsgId("12345678");
header.setTransactionId("123");
header.setProducerGroup("testTransactionGroup");
header.setTranStateTableOffset(1234L);
return header;
}
private MessageExtBrokerInner createMessageBrokerInner(long queueOffset, String topic, String body) {
MessageExtBrokerInner inner = new MessageExtBrokerInner();
inner.setBornTimestamp(System.currentTimeMillis() - 80000);
inner.setTransactionId("123456123");
inner.setTopic(topic);
inner.setQueueOffset(queueOffset);
inner.setBody(body.getBytes());
inner.setMsgId("123456123");
inner.setQueueId(0);
inner.setTopic("hello");
return inner;
}
private MessageExtBrokerInner createMessageBrokerInner() {
return createMessageBrokerInner(1, "testTopic", "hello world");
}
}
...@@ -14,10 +14,15 @@ ...@@ -14,10 +14,15 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.rocketmq.client.producer; package org.apache.rocketmq.broker.util;
import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;
import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExt;
public interface TransactionCheckListener { public class LogTransactionalMessageCheckListener extends AbstractTransactionalMessageCheckListener {
LocalTransactionState checkLocalTransactionState(final MessageExt msg);
@Override
public void resolveDiscardMsg(MessageExt msgExt) {
}
} }
...@@ -15,38 +15,27 @@ ...@@ -15,38 +15,27 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.rocketmq.client.log; package org.apache.rocketmq.broker.util;
import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;
import org.junit.After; import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger;
import java.io.File; import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals; public class ServiceProviderTest {
public class ClientLoggerTest { @Test
public void loadTransactionMsgServiceTest() {
public static final String CLIENT_LOG_ROOT = "rocketmq.client.logRoot"; TransactionalMessageService transactionService = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_SERVICE_ID,
public static final String LOG_DIR; TransactionalMessageService.class);
assertThat(transactionService).isNotNull();
static {
LOG_DIR = System.getProperty(CLIENT_LOG_ROOT, System.getProperty("user.home") + "/logs/rocketmqlogs");
}
@After
public void cleanFiles() {
UtilAll.deleteFile(new File(LOG_DIR));
} }
// FIXME: Workaround for concrete implementation for slf4j, is there any better solution for all slf4j implementations in one class ? 2017/8/1
@Test @Test
public void testLog4j2() throws Exception { public void loadAbstractTransactionListenerTest() {
Logger logger = ClientLogger.getLog(); AbstractTransactionalMessageCheckListener listener = ServiceProvider.loadClass(ServiceProvider.TRANSACTION_LISTENER_ID,
AbstractTransactionalMessageCheckListener.class);
System.out.println(logger); assertThat(listener).isNotNull();
assertEquals("org.apache.logging.slf4j.Log4jLogger", logger.getClass().getName());
} }
} }
...@@ -14,45 +14,54 @@ ...@@ -14,45 +14,54 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.rocketmq.broker.util;
package org.apache.rocketmq.filtersrv.stats; import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;
import org.apache.rocketmq.broker.transaction.OperationResult;
import java.util.concurrent.Executors; import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
import java.util.concurrent.ScheduledExecutorService;
import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.stats.StatsItemSet; import org.apache.rocketmq.common.message.MessageExt;
import org.slf4j.Logger; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
import org.slf4j.LoggerFactory; import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.PutMessageResult;
public class FilterServerStatsManager { public class TransactionalMessageServiceImpl implements TransactionalMessageService {
private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
private final ScheduledExecutorService scheduledExecutorService = Executors
.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSStatsThread"));
// ConsumerGroup Get Nums @Override
private final StatsItemSet groupGetNums = new StatsItemSet("GROUP_GET_NUMS", public PutMessageResult prepareMessage(MessageExtBrokerInner messageInner) {
this.scheduledExecutorService, log); return null;
}
// ConsumerGroup Get Size @Override
private final StatsItemSet groupGetSize = new StatsItemSet("GROUP_GET_SIZE", public boolean deletePrepareMessage(MessageExt messageExt) {
this.scheduledExecutorService, log); return false;
}
public FilterServerStatsManager() { @Override
public OperationResult commitMessage(EndTransactionRequestHeader requestHeader) {
return null;
} }
public void start() { @Override
public OperationResult rollbackMessage(EndTransactionRequestHeader requestHeader) {
return null;
} }
public void shutdown() { @Override
this.scheduledExecutorService.shutdown(); public void check(long transactionTimeout, int transactionCheckMax, AbstractTransactionalMessageCheckListener listener) {
log.warn("check check!");
} }
public void incGroupGetNums(final String group, final String topic, final int incValue) { @Override
this.groupGetNums.addValue(topic + "@" + group, incValue, 1); public boolean open() {
return true;
} }
public void incGroupGetSize(final String group, final String topic, final int incValue) { @Override
this.groupGetSize.addValue(topic + "@" + group, incValue, 1); public void close() {
} }
} }
org.apache.rocketmq.broker.util.LogTransactionalMessageCheckListener
\ No newline at end of file
org.apache.rocketmq.broker.util.TransactionalMessageServiceImpl
\ No newline at end of file
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
<parent> <parent>
<groupId>org.apache.rocketmq</groupId> <groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-all</artifactId> <artifactId>rocketmq-all</artifactId>
<version>4.2.0</version> <version>4.4.0-SNAPSHOT</version>
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
...@@ -46,6 +46,7 @@ ...@@ -46,6 +46,7 @@
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId> <artifactId>slf4j-api</artifactId>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
......
...@@ -20,6 +20,7 @@ import org.apache.rocketmq.common.MixAll; ...@@ -20,6 +20,7 @@ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.common.RemotingUtil;
import org.apache.rocketmq.remoting.netty.TlsSystemConfig; import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
import org.apache.rocketmq.remoting.protocol.LanguageCode;
/** /**
* Client Common configuration * Client Common configuration
...@@ -48,6 +49,8 @@ public class ClientConfig { ...@@ -48,6 +49,8 @@ public class ClientConfig {
private boolean useTLS = TlsSystemConfig.tlsEnable; private boolean useTLS = TlsSystemConfig.tlsEnable;
private LanguageCode language = LanguageCode.JAVA;
public String buildMQClientId() { public String buildMQClientId() {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append(this.getClientIP()); sb.append(this.getClientIP());
...@@ -96,6 +99,7 @@ public class ClientConfig { ...@@ -96,6 +99,7 @@ public class ClientConfig {
this.unitName = cc.unitName; this.unitName = cc.unitName;
this.vipChannelEnabled = cc.vipChannelEnabled; this.vipChannelEnabled = cc.vipChannelEnabled;
this.useTLS = cc.useTLS; this.useTLS = cc.useTLS;
this.language = cc.language;
} }
public ClientConfig cloneClientConfig() { public ClientConfig cloneClientConfig() {
...@@ -111,6 +115,7 @@ public class ClientConfig { ...@@ -111,6 +115,7 @@ public class ClientConfig {
cc.unitName = unitName; cc.unitName = unitName;
cc.vipChannelEnabled = vipChannelEnabled; cc.vipChannelEnabled = vipChannelEnabled;
cc.useTLS = useTLS; cc.useTLS = useTLS;
cc.language = language;
return cc; return cc;
} }
...@@ -186,12 +191,20 @@ public class ClientConfig { ...@@ -186,12 +191,20 @@ public class ClientConfig {
this.useTLS = useTLS; this.useTLS = useTLS;
} }
public LanguageCode getLanguage() {
return language;
}
public void setLanguage(LanguageCode language) {
this.language = language;
}
@Override @Override
public String toString() { public String toString() {
return "ClientConfig [namesrvAddr=" + namesrvAddr + ", clientIP=" + clientIP + ", instanceName=" + instanceName return "ClientConfig [namesrvAddr=" + namesrvAddr + ", clientIP=" + clientIP + ", instanceName=" + instanceName
+ ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + ", pollNameServerInterval=" + pollNameServerInterval + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + ", pollNameServerInterval=" + pollNameServerInterval
+ ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval="
+ persistConsumerOffsetInterval + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled=" + persistConsumerOffsetInterval + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled="
+ vipChannelEnabled + ", useTLS=" + useTLS + "]"; + vipChannelEnabled + ", useTLS=" + useTLS + ", language=" + language.name() + "]";
} }
} }
...@@ -20,9 +20,9 @@ import java.util.Set; ...@@ -20,9 +20,9 @@ import java.util.Set;
import java.util.TreeSet; import java.util.TreeSet;
import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.slf4j.Logger;
public class MQHelper { public class MQHelper {
public static void resetOffsetByTimestamp( public static void resetOffsetByTimestamp(
...@@ -48,7 +48,7 @@ public class MQHelper { ...@@ -48,7 +48,7 @@ public class MQHelper {
final String consumerGroup, final String consumerGroup,
final String topic, final String topic,
final long timestamp) throws Exception { final long timestamp) throws Exception {
final Logger log = ClientLogger.getLog(); final InternalLogger log = ClientLogger.getLog();
DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup); DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup);
consumer.setInstanceName(instanceName); consumer.setInstanceName(instanceName);
......
...@@ -119,9 +119,9 @@ public class Validators { ...@@ -119,9 +119,9 @@ public class Validators {
} }
//whether the same with system reserved keyword //whether the same with system reserved keyword
if (topic.equals(MixAll.DEFAULT_TOPIC)) { if (topic.equals(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC)) {
throw new MQClientException( throw new MQClientException(
String.format("The topic[%s] is conflict with default topic.", topic), null); String.format("The topic[%s] is conflict with AUTO_CREATE_TOPIC_KEY_TOPIC.", topic), null);
} }
} }
} }
...@@ -26,15 +26,15 @@ import java.util.concurrent.TimeUnit; ...@@ -26,15 +26,15 @@ import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.slf4j.Logger;
/** /**
* Schedule service for pull consumer * Schedule service for pull consumer
*/ */
public class MQPullConsumerScheduleService { public class MQPullConsumerScheduleService {
private final Logger log = ClientLogger.getLog(); private final InternalLogger log = ClientLogger.getLog();
private final MessageQueueListener messageQueueListener = new MessageQueueListenerImpl(); private final MessageQueueListener messageQueueListener = new MessageQueueListenerImpl();
private final ConcurrentMap<MessageQueue, PullTaskImpl> taskTable = private final ConcurrentMap<MessageQueue, PullTaskImpl> taskTable =
new ConcurrentHashMap<MessageQueue, PullTaskImpl>(); new ConcurrentHashMap<MessageQueue, PullTaskImpl>();
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.client.consumer.rebalance;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.logging.InternalLogger;
/**
* An allocate strategy proxy for based on machine room nearside priority. An actual allocate strategy can be
* specified.
*
* If any consumer is alive in a machine room, the message queue of the broker which is deployed in the same machine
* should only be allocated to those. Otherwise, those message queues can be shared along all consumers since there are
* no alive consumer to monopolize them.
*/
public class AllocateMachineRoomNearby implements AllocateMessageQueueStrategy {
private final InternalLogger log = ClientLogger.getLog();
private final AllocateMessageQueueStrategy allocateMessageQueueStrategy;//actual allocate strategy
private final MachineRoomResolver machineRoomResolver;
public AllocateMachineRoomNearby(AllocateMessageQueueStrategy allocateMessageQueueStrategy,
MachineRoomResolver machineRoomResolver) throws NullPointerException {
if (allocateMessageQueueStrategy == null) {
throw new NullPointerException("allocateMessageQueueStrategy is null");
}
if (machineRoomResolver == null) {
throw new NullPointerException("machineRoomResolver is null");
}
this.allocateMessageQueueStrategy = allocateMessageQueueStrategy;
this.machineRoomResolver = machineRoomResolver;
}
@Override
public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
List<String> cidAll) {
if (currentCID == null || currentCID.length() < 1) {
throw new IllegalArgumentException("currentCID is empty");
}
if (mqAll == null || mqAll.isEmpty()) {
throw new IllegalArgumentException("mqAll is null or mqAll empty");
}
if (cidAll == null || cidAll.isEmpty()) {
throw new IllegalArgumentException("cidAll is null or cidAll empty");
}
List<MessageQueue> result = new ArrayList<MessageQueue>();
if (!cidAll.contains(currentCID)) {
log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}",
consumerGroup,
currentCID,
cidAll);
return result;
}
//group mq by machine room
Map<String/*machine room */, List<MessageQueue>> mr2Mq = new TreeMap<String, List<MessageQueue>>();
for (MessageQueue mq : mqAll) {
String brokerMachineRoom = machineRoomResolver.brokerDeployIn(mq);
if (StringUtils.isNoneEmpty(brokerMachineRoom)) {
if (mr2Mq.get(brokerMachineRoom) == null) {
mr2Mq.put(brokerMachineRoom, new ArrayList<MessageQueue>());
}
mr2Mq.get(brokerMachineRoom).add(mq);
} else {
throw new IllegalArgumentException("Machine room is null for mq " + mq);
}
}
//group consumer by machine room
Map<String/*machine room */, List<String/*clientId*/>> mr2c = new TreeMap<String, List<String>>();
for (String cid : cidAll) {
String consumerMachineRoom = machineRoomResolver.consumerDeployIn(cid);
if (StringUtils.isNoneEmpty(consumerMachineRoom)) {
if (mr2c.get(consumerMachineRoom) == null) {
mr2c.put(consumerMachineRoom, new ArrayList<String>());
}
mr2c.get(consumerMachineRoom).add(cid);
} else {
throw new IllegalArgumentException("Machine room is null for consumer id " + cid);
}
}
List<MessageQueue> allocateResults = new ArrayList<MessageQueue>();
//1.allocate the mq that deploy in the same machine room with the current consumer
String currentMachineRoom = machineRoomResolver.consumerDeployIn(currentCID);
List<MessageQueue> mqInThisMachineRoom = mr2Mq.remove(currentMachineRoom);
List<String> consumerInThisMachineRoom = mr2c.get(currentMachineRoom);
if (mqInThisMachineRoom != null && !mqInThisMachineRoom.isEmpty()) {
allocateResults.addAll(allocateMessageQueueStrategy.allocate(consumerGroup, currentCID, mqInThisMachineRoom, consumerInThisMachineRoom));
}
//2.allocate the rest mq to each machine room if there are no consumer alive in that machine room
for (String machineRoom : mr2Mq.keySet()) {
if (!mr2c.containsKey(machineRoom)) { // no alive consumer in the corresponding machine room, so all consumers share these queues
allocateResults.addAll(allocateMessageQueueStrategy.allocate(consumerGroup, currentCID, mr2Mq.get(machineRoom), cidAll));
}
}
return allocateResults;
}
@Override
public String getName() {
return "MACHINE_ROOM_NEARBY" + "-" + allocateMessageQueueStrategy.getName();
}
/**
* A resolver object to determine which machine room do the message queues or clients are deployed in.
*
* AllocateMachineRoomNearby will use the results to group the message queues and clients by machine room.
*
* The result returned from the implemented method CANNOT be null.
*/
public interface MachineRoomResolver {
String brokerDeployIn(MessageQueue messageQueue);
String consumerDeployIn(String clientID);
}
}
...@@ -20,14 +20,14 @@ import java.util.ArrayList; ...@@ -20,14 +20,14 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.slf4j.Logger;
/** /**
* Average Hashing queue algorithm * Average Hashing queue algorithm
*/ */
public class AllocateMessageQueueAveragely implements AllocateMessageQueueStrategy { public class AllocateMessageQueueAveragely implements AllocateMessageQueueStrategy {
private final Logger log = ClientLogger.getLog(); private final InternalLogger log = ClientLogger.getLog();
@Override @Override
public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll, public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
......
...@@ -20,14 +20,14 @@ import java.util.ArrayList; ...@@ -20,14 +20,14 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.slf4j.Logger;
/** /**
* Cycle average Hashing queue algorithm * Cycle average Hashing queue algorithm
*/ */
public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy { public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy {
private final Logger log = ClientLogger.getLog(); private final InternalLogger log = ClientLogger.getLog();
@Override @Override
public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll, public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
......
...@@ -24,14 +24,14 @@ import org.apache.rocketmq.client.log.ClientLogger; ...@@ -24,14 +24,14 @@ import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter; import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter;
import org.apache.rocketmq.common.consistenthash.HashFunction; import org.apache.rocketmq.common.consistenthash.HashFunction;
import org.apache.rocketmq.common.consistenthash.Node; import org.apache.rocketmq.common.consistenthash.Node;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.slf4j.Logger;
/** /**
* Consistent Hashing queue algorithm * Consistent Hashing queue algorithm
*/ */
public class AllocateMessageQueueConsistentHash implements AllocateMessageQueueStrategy { public class AllocateMessageQueueConsistentHash implements AllocateMessageQueueStrategy {
private final Logger log = ClientLogger.getLog(); private final InternalLogger log = ClientLogger.getLog();
private final int virtualNodeCnt; private final int virtualNodeCnt;
private final HashFunction customHashFunction; private final HashFunction customHashFunction;
......
...@@ -31,9 +31,9 @@ import org.apache.rocketmq.client.log.ClientLogger; ...@@ -31,9 +31,9 @@ import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingException;
import org.slf4j.Logger;
/** /**
* Local storage implementation * Local storage implementation
...@@ -42,7 +42,7 @@ public class LocalFileOffsetStore implements OffsetStore { ...@@ -42,7 +42,7 @@ public class LocalFileOffsetStore implements OffsetStore {
public final static String LOCAL_OFFSET_STORE_DIR = System.getProperty( public final static String LOCAL_OFFSET_STORE_DIR = System.getProperty(
"rocketmq.client.localOffsetStoreDir", "rocketmq.client.localOffsetStoreDir",
System.getProperty("user.home") + File.separator + ".rocketmq_offsets"); System.getProperty("user.home") + File.separator + ".rocketmq_offsets");
private final static Logger log = ClientLogger.getLog(); private final static InternalLogger log = ClientLogger.getLog();
private final MQClientInstance mQClientFactory; private final MQClientInstance mQClientFactory;
private final String groupName; private final String groupName;
private final String storePath; private final String storePath;
......
...@@ -30,17 +30,17 @@ import org.apache.rocketmq.client.impl.factory.MQClientInstance; ...@@ -30,17 +30,17 @@ import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader;
import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader;
import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingException;
import org.slf4j.Logger;
/** /**
* Remote storage implementation * Remote storage implementation
*/ */
public class RemoteBrokerOffsetStore implements OffsetStore { public class RemoteBrokerOffsetStore implements OffsetStore {
private final static Logger log = ClientLogger.getLog(); private final static InternalLogger log = ClientLogger.getLog();
private final MQClientInstance mQClientFactory; private final MQClientInstance mQClientFactory;
private final String groupName; private final String groupName;
private ConcurrentMap<MessageQueue, AtomicLong> offsetTable = private ConcurrentMap<MessageQueue, AtomicLong> offsetTable =
......
...@@ -24,6 +24,7 @@ import org.apache.rocketmq.client.impl.factory.MQClientInstance; ...@@ -24,6 +24,7 @@ import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.impl.producer.MQProducerInner; import org.apache.rocketmq.client.impl.producer.MQProducerInner;
import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExt;
...@@ -44,10 +45,10 @@ import org.apache.rocketmq.remoting.common.RemotingHelper; ...@@ -44,10 +45,10 @@ import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.slf4j.Logger;
public class ClientRemotingProcessor implements NettyRequestProcessor { public class ClientRemotingProcessor implements NettyRequestProcessor {
private final Logger log = ClientLogger.getLog(); private final InternalLogger log = ClientLogger.getLog();
private final MQClientInstance mqClientFactory; private final MQClientInstance mqClientFactory;
public ClientRemotingProcessor(final MQClientInstance mqClientFactory) { public ClientRemotingProcessor(final MQClientInstance mqClientFactory) {
...@@ -90,6 +91,10 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { ...@@ -90,6 +91,10 @@ public class ClientRemotingProcessor implements NettyRequestProcessor {
final ByteBuffer byteBuffer = ByteBuffer.wrap(request.getBody()); final ByteBuffer byteBuffer = ByteBuffer.wrap(request.getBody());
final MessageExt messageExt = MessageDecoder.decode(byteBuffer); final MessageExt messageExt = MessageDecoder.decode(byteBuffer);
if (messageExt != null) { if (messageExt != null) {
String transactionId = messageExt.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);
if (null != transactionId && !"".equals(transactionId)) {
messageExt.setTransactionId(transactionId);
}
final String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); final String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);
if (group != null) { if (group != null) {
MQProducerInner producer = this.mqClientFactory.selectProducer(group); MQProducerInner producer = this.mqClientFactory.selectProducer(group);
......
...@@ -22,11 +22,11 @@ import java.util.concurrent.atomic.AtomicInteger; ...@@ -22,11 +22,11 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.RPCHook;
import org.slf4j.Logger;
public class MQClientManager { public class MQClientManager {
private final static Logger log = ClientLogger.getLog(); private final static InternalLogger log = ClientLogger.getLog();
private static MQClientManager instance = new MQClientManager(); private static MQClientManager instance = new MQClientManager();
private AtomicInteger factoryIndexGenerator = new AtomicInteger(); private AtomicInteger factoryIndexGenerator = new AtomicInteger();
private ConcurrentMap<String/* clientId */, MQClientInstance> factoryTable = private ConcurrentMap<String/* clientId */, MQClientInstance> factoryTable =
......
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册