提交 498e8a52 编写于 作者: wu-sheng's avatar wu-sheng

Isolate NetworkAddressRegisterService from ApplicationRegisterService

上级 ce1ba148
......@@ -29,8 +29,6 @@ import org.apache.skywalking.apm.collector.core.util.TimeBucketUtils;
import org.apache.skywalking.apm.collector.server.grpc.GRPCHandler;
import org.apache.skywalking.apm.network.proto.ApplicationInstance;
import org.apache.skywalking.apm.network.proto.ApplicationInstanceMapping;
import org.apache.skywalking.apm.network.proto.ApplicationInstanceRecover;
import org.apache.skywalking.apm.network.proto.Downstream;
import org.apache.skywalking.apm.network.proto.InstanceDiscoveryServiceGrpc;
import org.apache.skywalking.apm.network.proto.OSInfo;
import org.slf4j.Logger;
......@@ -60,14 +58,6 @@ public class InstanceDiscoveryServiceHandler extends InstanceDiscoveryServiceGrp
responseObserver.onCompleted();
}
@Override
public void registerRecover(ApplicationInstanceRecover request, StreamObserver<Downstream> responseObserver) {
long timeBucket = TimeBucketUtils.INSTANCE.getSecondTimeBucket(request.getRegisterTime());
instanceIDService.recover(request.getApplicationInstanceId(), request.getApplicationId(), timeBucket, buildOsInfo(request.getOsinfo()));
responseObserver.onNext(Downstream.newBuilder().build());
responseObserver.onCompleted();
}
private String buildOsInfo(OSInfo osinfo) {
JsonObject osInfoJson = new JsonObject();
osInfoJson.addProperty("osName", osinfo.getOsName());
......
......@@ -7,6 +7,7 @@ import "KeyWithIntegerValue.proto";
//register service for ApplicationCode, this service is called when service starts.
service ApplicationRegisterService {
//TODO: TODO: `batchRegister` should be replaces by applicationCodeRegister
rpc batchRegister (Applications) returns (ApplicationMappings) {
}
}
......
......@@ -5,17 +5,14 @@ option java_package = "org.apache.skywalking.apm.network.proto";
import "Downstream.proto";
//discovery service for application instance, this service is called when application starts
//or http client connection switch to another collector server instance
service InstanceDiscoveryService {
//TODO: need rename, `register` is a key word.
rpc register (ApplicationInstance) returns (ApplicationInstanceMapping) {
}
rpc heartbeat (ApplicationInstanceHeartbeat) returns (Downstream) {
}
rpc registerRecover (ApplicationInstanceRecover) returns (Downstream) {
}
}
message ApplicationInstance {
......
syntax = "proto3";
option java_multiple_files = true;
option java_package = "org.apache.skywalking.apm.network.proto";
import "KeyWithIntegerValue.proto";
//register service for ApplicationCode, this service is called when service starts.
service NetworkAddressRegisterService {
rpc batchRegister (NetworkAddresses) returns (NetworkAddressMappings) {
}
}
message NetworkAddresses {
repeated string addresses = 1;
}
message NetworkAddressMappings {
repeated KeyWithIntegerValue addressIds = 1;
}
......@@ -323,7 +323,7 @@ public class TracingContext implements AbstractTracerContext {
exitSpan = parentSpan;
} else {
final int parentSpanId = parentSpan == null ? -1 : parentSpan.getSpanId();
exitSpan = (AbstractSpan)DictionaryManager.findApplicationCodeSection()
exitSpan = (AbstractSpan)DictionaryManager.findNetworkAddressSection()
.find(remotePeer).doInCondition(
new PossibleFound.FoundAndObtain() {
@Override
......
......@@ -24,10 +24,10 @@ package org.apache.skywalking.apm.agent.core.dictionary;
*/
public class DictionaryManager {
/**
* @return {@link ApplicationDictionary} to find application id for application code and network address.
* @return {@link NetworkAddressDictionary} to find application id for application code and network address.
*/
public static ApplicationDictionary findApplicationCodeSection() {
return ApplicationDictionary.INSTANCE;
public static NetworkAddressDictionary findNetworkAddressSection() {
return NetworkAddressDictionary.INSTANCE;
}
/**
......
......@@ -23,42 +23,42 @@ import io.netty.util.internal.ConcurrentSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.skywalking.apm.network.proto.ApplicationMappings;
import org.apache.skywalking.apm.network.proto.ApplicationRegisterServiceGrpc;
import org.apache.skywalking.apm.network.proto.Applications;
import org.apache.skywalking.apm.network.proto.KeyWithIntegerValue;
import org.apache.skywalking.apm.network.proto.NetworkAddressMappings;
import org.apache.skywalking.apm.network.proto.NetworkAddressRegisterServiceGrpc;
import org.apache.skywalking.apm.network.proto.NetworkAddresses;
import static org.apache.skywalking.apm.agent.core.conf.Config.Dictionary.APPLICATION_CODE_BUFFER_SIZE;
/**
* Map of application id to application code, which is from the collector side.
* Map of network address id to network literal address, which is from the collector side.
*
* @author wusheng
*/
public enum ApplicationDictionary {
public enum NetworkAddressDictionary {
INSTANCE;
private Map<String, Integer> applicationDictionary = new ConcurrentHashMap<String, Integer>();
private Set<String> unRegisterApplications = new ConcurrentSet<String>();
public PossibleFound find(String applicationCode) {
Integer applicationId = applicationDictionary.get(applicationCode);
public PossibleFound find(String networkAddress) {
Integer applicationId = applicationDictionary.get(networkAddress);
if (applicationId != null) {
return new Found(applicationId);
} else {
if (applicationDictionary.size() + unRegisterApplications.size() < APPLICATION_CODE_BUFFER_SIZE) {
unRegisterApplications.add(applicationCode);
unRegisterApplications.add(networkAddress);
}
return new NotFound();
}
}
public void syncRemoteDictionary(
ApplicationRegisterServiceGrpc.ApplicationRegisterServiceBlockingStub applicationRegisterServiceBlockingStub) {
NetworkAddressRegisterServiceGrpc.NetworkAddressRegisterServiceBlockingStub networkAddressRegisterServiceBlockingStub) {
if (unRegisterApplications.size() > 0) {
ApplicationMappings applicationMapping = applicationRegisterServiceBlockingStub.batchRegister(
Applications.newBuilder().addAllApplicationCodes(unRegisterApplications).build());
if (applicationMapping.getApplicationsCount() > 0) {
for (KeyWithIntegerValue keyWithIntegerValue : applicationMapping.getApplicationsList()) {
NetworkAddressMappings networkAddressMappings = networkAddressRegisterServiceBlockingStub.batchRegister(
NetworkAddresses.newBuilder().addAllAddresses(unRegisterApplications).build());
if (networkAddressMappings.getAddressIdsCount() > 0) {
for (KeyWithIntegerValue keyWithIntegerValue : networkAddressMappings.getAddressIdsList()) {
unRegisterApplications.remove(keyWithIntegerValue.getKey());
applicationDictionary.put(keyWithIntegerValue.getKey(), keyWithIntegerValue.getValue());
}
......
......@@ -16,7 +16,6 @@
*
*/
package org.apache.skywalking.apm.agent.core.remote;
import io.grpc.ManagedChannel;
......@@ -24,28 +23,28 @@ import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import org.apache.skywalking.apm.agent.core.conf.RemoteDownstreamConfig;
import org.apache.skywalking.apm.agent.core.context.TracingContext;
import org.apache.skywalking.apm.agent.core.dictionary.ApplicationDictionary;
import org.apache.skywalking.apm.agent.core.os.OSUtil;
import org.apache.skywalking.apm.agent.core.boot.BootService;
import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory;
import org.apache.skywalking.apm.agent.core.boot.ServiceManager;
import org.apache.skywalking.apm.agent.core.conf.Config;
import org.apache.skywalking.apm.agent.core.conf.RemoteDownstreamConfig;
import org.apache.skywalking.apm.agent.core.context.TracingContext;
import org.apache.skywalking.apm.agent.core.context.TracingContextListener;
import org.apache.skywalking.apm.agent.core.context.trace.TraceSegment;
import org.apache.skywalking.apm.agent.core.dictionary.DictionaryUtil;
import org.apache.skywalking.apm.agent.core.dictionary.NetworkAddressDictionary;
import org.apache.skywalking.apm.agent.core.dictionary.OperationNameDictionary;
import org.apache.skywalking.apm.agent.core.logging.api.ILog;
import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
import org.apache.skywalking.apm.agent.core.os.OSUtil;
import org.apache.skywalking.apm.network.proto.ApplicationInstance;
import org.apache.skywalking.apm.network.proto.ApplicationInstanceHeartbeat;
import org.apache.skywalking.apm.network.proto.ApplicationInstanceMapping;
import org.apache.skywalking.apm.network.proto.ApplicationInstanceRecover;
import org.apache.skywalking.apm.network.proto.ApplicationMappings;
import org.apache.skywalking.apm.network.proto.ApplicationRegisterServiceGrpc;
import org.apache.skywalking.apm.network.proto.Applications;
import org.apache.skywalking.apm.network.proto.InstanceDiscoveryServiceGrpc;
import org.apache.skywalking.apm.network.proto.NetworkAddressRegisterServiceGrpc;
import org.apache.skywalking.apm.network.proto.ServiceNameDiscoveryServiceGrpc;
/**
......@@ -59,8 +58,8 @@ public class AppAndServiceRegisterClient implements BootService, GRPCChannelList
private volatile ApplicationRegisterServiceGrpc.ApplicationRegisterServiceBlockingStub applicationRegisterServiceBlockingStub;
private volatile InstanceDiscoveryServiceGrpc.InstanceDiscoveryServiceBlockingStub instanceDiscoveryServiceBlockingStub;
private volatile ServiceNameDiscoveryServiceGrpc.ServiceNameDiscoveryServiceBlockingStub serviceNameDiscoveryServiceBlockingStub;
private volatile NetworkAddressRegisterServiceGrpc.NetworkAddressRegisterServiceBlockingStub networkAddressRegisterServiceBlockingStub;
private volatile ScheduledFuture<?> applicationRegisterFuture;
private volatile boolean needRegisterRecover = false;
private volatile long lastSegmentTime = -1;
@Override
......@@ -69,10 +68,8 @@ public class AppAndServiceRegisterClient implements BootService, GRPCChannelList
ManagedChannel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getManagedChannel();
applicationRegisterServiceBlockingStub = ApplicationRegisterServiceGrpc.newBlockingStub(channel);
instanceDiscoveryServiceBlockingStub = InstanceDiscoveryServiceGrpc.newBlockingStub(channel);
if (RemoteDownstreamConfig.Agent.APPLICATION_INSTANCE_ID != DictionaryUtil.nullValue()) {
needRegisterRecover = true;
}
serviceNameDiscoveryServiceBlockingStub = ServiceNameDiscoveryServiceGrpc.newBlockingStub(channel);
networkAddressRegisterServiceBlockingStub = NetworkAddressRegisterServiceGrpc.newBlockingStub(channel);
} else {
applicationRegisterServiceBlockingStub = null;
instanceDiscoveryServiceBlockingStub = null;
......@@ -105,13 +102,14 @@ public class AppAndServiceRegisterClient implements BootService, GRPCChannelList
@Override
public void run() {
logger.debug("AppAndServiceRegisterClient running, status:{}.",status);
logger.debug("AppAndServiceRegisterClient running, status:{}.", status);
boolean shouldTry = true;
while (GRPCChannelStatus.CONNECTED.equals(status) && shouldTry) {
shouldTry = false;
try {
if (RemoteDownstreamConfig.Agent.APPLICATION_ID == DictionaryUtil.nullValue()) {
if (applicationRegisterServiceBlockingStub != null) {
//TODO: `batchRegister` should be replaces by applicationCodeRegister
ApplicationMappings applicationMapping = applicationRegisterServiceBlockingStub.batchRegister(
Applications.newBuilder().addApplicationCodes(Config.Agent.APPLICATION_CODE).build());
if (applicationMapping.getApplicationsCount() > 0) {
......@@ -134,24 +132,14 @@ public class AppAndServiceRegisterClient implements BootService, GRPCChannelList
= instanceMapping.getApplicationInstanceId();
}
} else {
if (needRegisterRecover) {
instanceDiscoveryServiceBlockingStub.registerRecover(ApplicationInstanceRecover.newBuilder()
.setApplicationId(RemoteDownstreamConfig.Agent.APPLICATION_ID)
if (lastSegmentTime - System.currentTimeMillis() > 60 * 1000) {
instanceDiscoveryServiceBlockingStub.heartbeat(ApplicationInstanceHeartbeat.newBuilder()
.setApplicationInstanceId(RemoteDownstreamConfig.Agent.APPLICATION_INSTANCE_ID)
.setRegisterTime(System.currentTimeMillis())
.setOsinfo(OSUtil.buildOSInfo())
.setHeartbeatTime(System.currentTimeMillis())
.build());
needRegisterRecover = false;
} else {
if (lastSegmentTime - System.currentTimeMillis() > 60 * 1000) {
instanceDiscoveryServiceBlockingStub.heartbeat(ApplicationInstanceHeartbeat.newBuilder()
.setApplicationInstanceId(RemoteDownstreamConfig.Agent.APPLICATION_INSTANCE_ID)
.setHeartbeatTime(System.currentTimeMillis())
.build());
}
}
ApplicationDictionary.INSTANCE.syncRemoteDictionary(applicationRegisterServiceBlockingStub);
NetworkAddressDictionary.INSTANCE.syncRemoteDictionary(networkAddressRegisterServiceBlockingStub);
OperationNameDictionary.INSTANCE.syncRemoteDictionary(serviceNameDiscoveryServiceBlockingStub);
}
}
......
# Trace Data Protocol 中文
Trace Data Protocol协议,也就是探针与Collector间通讯协议
## 前言
## 概述
此协议包含了Agent上行/下行数据的格式,可用于定制开发,或者探针的多语言扩展
## 协议类型
对外同时提供gRPC和HTTP RESTFul两种类型的协议。从效率上,我们推荐使用gRPC
### 协议版本
v1.1
### 协议类型
* 服务发现使用http服务
* 注册和数据上行服务同时支持gRPC和HTTP服务
#### gRPC协议定义文件
[gRPC proto files](../../apm-protocol/apm-network/src/main/proto)
## Collector服务发现协议
### 简介
**Collector服务发现协议是探针启动时,第一个调用的服务。**通过服务,查找对应的gRPC服务地址与端口列表,并在由客户端选择其中任意一个作为服务端。此服务需周期性调用,确保探针本地的服务端口列表是准确有效的。
**Collector服务发现协议是探针启动时,第一个调用的服务。** 通过服务,查找可用的gRPC服务地址列表,并在由客户端选择其中任意一个作为服务端。
此服务建议周期性调用,确保探针本地的服务端口列表是准确有效的。
### 协议类型
HTTP GET
......@@ -26,41 +34,17 @@ JSON数组,数组的每个元素,为一个有效的gRPC服务地址。
## 应用注册服务
### 简介
应用注册服务,是将手动设计的applicationCode,以及ip:port沟通的服务地址,转换成数字的服务。此服务会在后续的传输过程中,有效降低网络带宽需求。
### 协议类型
gRPC服务
应用注册服务,是将applicationCode,以及ip:port构成的服务地址,转换成数字ID的服务。
此服务会在后续的传输过程中,有效降低网络带宽需求。
### 协议内容
[gRPC service define](../..apm-protocol/apm-network/src/main/proto/ApplicationRegisterService.proto)
```proto
syntax = "proto3";
option java_multiple_files = true;
option java_package = "org.apache.skywalking.apm.network.proto";
import "KeyWithIntegerValue.proto";
//register service for ApplicationCode, this service is called when service starts.
service ApplicationRegisterService {
rpc batchRegister (Applications) returns (ApplicationMappings) {
}
}
message Applications {
repeated string applicationCodes = 1;
}
message ApplicationMappings {
repeated KeyWithIntegerValue applications = 1;
}
```
- 首次调用时,applicationCode为客户端设置的应用名(显示在拓扑图和应用列表上的名字)。之后随着追踪过程,会上报此应用相关的周边服务的`ip:port`地址列表
- KeyWithIntegerValue 返回,key为上报的applicationCodeip:port地址,value为对应的id。applicationCode对应的返回id,在后续协议中,被称为applicationId。
- KeyWithIntegerValue 返回,key为上报的applicationCodeip:port地址,value为对应的id。applicationCode对应的返回id,在后续协议中,被称为applicationId。
- 此服务按需调用,本地无法找到ip:port对应的id时,可异步发起调用。
- 获取applicationId的操作是必选。后续追踪数据依赖此id
- 获取ip:port对应的id是可选,但是完成id设置,会有效提高collector处理效率,降低网络消耗。
- 获取ip:port对应的id是可选,使用id,会有效提高collector处理效率,降低网络消耗。
## 应用实例发现服务
### 简介
......@@ -71,102 +55,24 @@ gRPC服务
### 实例注册服务
[gRPC service define](../../apm-protocol/apm-network/src/main/proto/DiscoveryService.proto#L11-L12)
```proto
service InstanceDiscoveryService {
rpc register (ApplicationInstance) returns (ApplicationInstanceMapping) {
}
}
message ApplicationInstance {
int32 applicationId = 1;
string agentUUID = 2;
int64 registerTime = 3;
OSInfo osinfo = 4;
}
message OSInfo {
string osName = 1;
string hostname = 2;
int32 processNo = 3;
repeated string ipv4s = 4;
}
message ApplicationInstanceMapping {
int32 applicationId = 1;
int32 applicationInstanceId = 2;
}
```
- agentUUID 由探针生成,需保持唯一性,推荐使用UUID算法。并在应用重启前保持不变
- applicationId 由**应用注册服务**获取。
- 服务端返回应用实例id,applicationInstanceId 。后续上报服务使用实例id标识。
### 实例心跳服务
[gRPC service define](../../apm-protocol/apm-network/src/main/proto/DiscoveryService.proto#L14-L15)
```proto
service InstanceDiscoveryService {
rpc heartbeat (ApplicationInstanceHeartbeat) returns (Downstream) {
}
}
message ApplicationInstanceHeartbeat {
int32 applicationInstanceId = 1;
int64 heartbeatTime = 2;
}
```
- 心跳服务每分钟上报一次。
- 如一分钟内有segment数据上报,则可不必上报心跳。
### 实例注册重连服务
https://github.com/apache/incubator-skywalking/blob/master/apm-network/src/main/proto/DiscoveryService.proto#L17-L18
```proto
service InstanceDiscoveryService {
rpc registerRecover (ApplicationInstanceRecover) returns (Downstream) {
}
}
message ApplicationInstanceRecover {
int32 applicationId = 1;
int32 applicationInstanceId = 2;
int64 registerTime = 3;
OSInfo osinfo = 4;
}
```
- 应用重连服务于**应用注册服务**类似,在gRPC发生重连,并再次连接成功后发送。需包含通过**应用注册服务**获取的applicationInstanceId。
- 如果一分钟内有segment数据上报,则可不必上报心跳。
## 服务名注册发现服务
### 简介
服务名注册发现服务,是将应用内的服务名(operationName)替换为id的服务。
### 协议类型
gRPC服务
### 协议内容
[gRPC service define](../../apm-protocol/apm-network/src/main/proto/DiscoveryService.proto#L53-L74)
```proto
//discovery service for ServiceName by Network address or application code
service ServiceNameDiscoveryService {
rpc discovery (ServiceNameCollection) returns (ServiceNameMappingCollection) {
}
}
message ServiceNameCollection {
repeated ServiceNameElement elements = 1;
}
message ServiceNameMappingCollection {
repeated ServiceNameMappingElement elements = 1;
}
message ServiceNameMappingElement {
int32 serviceId = 1;
ServiceNameElement element = 2;
}
message ServiceNameElement {
string serviceName = 1;
int32 applicationId = 2;
}
```
- 可选服务,可有效降低网络消耗,推荐实现。注意,由于部分应用存在URI中夹带参数的情况,请注意限制探针内的缓存容量,防止内存溢出。
- ServiceNameElement中,applicationId为当前applicationCode对应的id。serviceName一般为对应span的operationName
......@@ -174,177 +80,16 @@ message ServiceNameElement {
### 简介
上报当前实例的JVM信息,每秒上报一次。
### 协议类型
gRPC服务
### 协议内容
[gRPC service define](../../apm-protocol/apm-network/src/main/proto/JVMMetricsService.proto)
```proto
syntax = "proto3";
option java_multiple_files = true;
option java_package = "org.apache.skywalking.apm.network.proto";
import "Downstream.proto";
service JVMMetricsService {
rpc collect (JVMMetrics) returns (Downstream) {
}
}
message JVMMetrics {
repeated JVMMetric metrics = 1;
int64 applicationInstanceId = 2;
}
message JVMMetric {
int64 time = 1;
CPU cpu = 2;
repeated Memory memory = 3;
repeated MemoryPool memoryPool = 4;
repeated GC gc = 5;
}
message CPU {
double usagePercent = 2;
}
message Memory {
bool isHeap = 1;
int64 init = 2;
int64 max = 3;
int64 used = 4;
int64 committed = 5;
}
message MemoryPool {
PoolType type = 1;
bool isHeap = 2;
int64 init = 3;
int64 max = 4;
int64 used = 5;
int64 commited = 6;
}
enum PoolType {
CODE_CACHE_USAGE = 0;
NEWGEN_USAGE = 1;
OLDGEN_USAGE = 2;
SURVIVOR_USAGE = 3;
PERMGEN_USAGE = 4;
METASPACE_USAGE = 5;
}
message GC {
GCPhrase phrase = 1;
int64 count = 2;
int64 time = 3;
}
enum GCPhrase {
NEW = 0;
OLD = 1;
}
```
## TraceSegment上报服务
### 简介
上报调用链信息
### 协议类型
gRPC服务
### 协议内容
[gRPC service define](../../apm-protocol/apm-network/src/main/proto/TraceSegmentService.proto)
```proto
syntax = "proto3";
option java_multiple_files = true;
option java_package = "org.apache.skywalking.apm.network.proto";
import "Downstream.proto";
import "KeyWithStringValue.proto";
service TraceSegmentService {
rpc collect (stream UpstreamSegment) returns (Downstream) {
}
}
message UpstreamSegment {
repeated UniqueId globalTraceIds = 1;
bytes segment = 2; // the byte array of TraceSegmentObject
}
message UniqueId {
repeated int64 idParts = 1;
}
message TraceSegmentObject {
UniqueId traceSegmentId = 1;
repeated SpanObject spans = 2;
int32 applicationId = 3;
int32 applicationInstanceId = 4;
bool isSizeLimited = 5;
}
message TraceSegmentReference {
RefType refType = 1;
UniqueId parentTraceSegmentId = 2;
int32 parentSpanId = 3;
int32 parentApplicationInstanceId = 4;
string networkAddress = 5;
int32 networkAddressId = 6;
int32 entryApplicationInstanceId = 7;
string entryServiceName = 8;
int32 entryServiceId = 9;
string parentServiceName = 10;
int32 parentServiceId = 11;
}
message SpanObject {
int32 spanId = 1;
int32 parentSpanId = 2;
int64 startTime = 3;
int64 endTime = 4;
repeated TraceSegmentReference refs = 5;
int32 operationNameId = 6;
string operationName = 7;
int32 peerId = 8;
string peer = 9;
SpanType spanType = 10;
SpanLayer spanLayer = 11;
int32 componentId = 12;
string component = 13;
bool isError = 14;
repeated KeyWithStringValue tags = 15;
repeated LogMessage logs = 16;
}
enum RefType {
CrossProcess = 0;
CrossThread = 1;
}
enum SpanType {
Entry = 0;
Exit = 1;
Local = 2;
}
enum SpanLayer {
Unknown = 0;
Database = 1;
RPCFramework = 2;
Http = 3;
MQ = 4;
Cache = 5;
}
message LogMessage {
int64 time = 1;
repeated KeyWithStringValue data = 2;
}
```
- UniqueId为segment或者globalTraceId的数字表示。由3个long组成,1)applicationInstanceId,2)当前线程id,3)当前时间戳*10000 + seq(0-10000自循环)
- Span的数据,请参考[插件开发规范](Plugin-Development-Guide-CN.md)
- 以下id和名称根据注册返回结果,优先上报id,无法获取id时,再上传name。参考之前的应用和服务注册章节。
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册