Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
b9450007
TDengine
项目概览
taosdata
/
TDengine
1 年多 前同步成功
通知
1185
Star
22016
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
b9450007
编写于
1月 11, 2022
作者:
dengyihao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add libuv
上级
3077a244
变更
10
显示空白变更内容
内联
并排
Showing
10 changed file
with
721 addition
and
547 deletion
+721
-547
cmake/libuv_CMakeLists.txt.in
cmake/libuv_CMakeLists.txt.in
+4
-4
source/libs/transport/CMakeLists.txt
source/libs/transport/CMakeLists.txt
+16
-1
source/libs/transport/inc/rpcCache.h
source/libs/transport/inc/rpcCache.h
+2
-0
source/libs/transport/inc/rpcHead.h
source/libs/transport/inc/rpcHead.h
+32
-28
source/libs/transport/inc/rpcTcp.h
source/libs/transport/inc/rpcTcp.h
+9
-4
source/libs/transport/inc/transportInt.h
source/libs/transport/inc/transportInt.h
+6
-1
source/libs/transport/src/rpcCache.c
source/libs/transport/src/rpcCache.c
+30
-23
source/libs/transport/src/rpcMain.c
source/libs/transport/src/rpcMain.c
+508
-379
source/libs/transport/src/rpcTcp.c
source/libs/transport/src/rpcTcp.c
+72
-68
source/libs/transport/src/rpcUdp.c
source/libs/transport/src/rpcUdp.c
+42
-39
未找到文件。
cmake/libuv_CMakeLists.txt.in
浏览文件 @
b9450007
...
...
@@ -4,7 +4,7 @@ ExternalProject_Add(libuv
GIT_REPOSITORY https://github.com/libuv/libuv.git
GIT_TAG v1.42.0
SOURCE_DIR "${CMAKE_CONTRIB_DIR}/libuv"
BINARY_DIR ""
BINARY_DIR "
${CMAKE_CONTRIB_DIR}/libuv
"
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
...
...
source/libs/transport/CMakeLists.txt
浏览文件 @
b9450007
...
...
@@ -13,3 +13,18 @@ target_link_libraries(
PUBLIC util
PUBLIC common
)
if
(
${
BUILD_WITH_UV
}
)
target_include_directories
(
transport
PUBLIC
"
${
CMAKE_SOURCE_DIR
}
/contrib/libuv/include"
)
#LINK_DIRECTORIES("${CMAKE_SOURCE_DIR}/debug/contrib/libuv")
target_link_libraries
(
transport
PUBLIC uv_a
)
add_definitions
(
-DUSE_UV
)
endif
(
${
BUILD_WITH_UV
}
)
source/libs/transport/inc/rpcCache.h
浏览文件 @
b9450007
...
...
@@ -16,6 +16,8 @@
#ifndef TDENGINE_RPC_CACHE_H
#define TDENGINE_RPC_CACHE_H
#include <stdint.h>
#ifdef __cplusplus
extern
"C"
{
#endif
...
...
source/libs/transport/inc/rpcHead.h
浏览文件 @
b9450007
...
...
@@ -16,33 +16,38 @@
#ifndef TDENGINE_RPCHEAD_H
#define TDENGINE_RPCHEAD_H
#include <tdef.h>
#ifdef __cplusplus
extern
"C"
{
#endif
#ifdef USE_UV
#else
#define RPC_CONN_TCP 2
extern
int
tsRpcOverhead
;
typedef
struct
{
void
*
msg
;
void
*
msg
;
int
msgLen
;
uint32_t
ip
;
uint16_t
port
;
int
connType
;
void
*
shandle
;
void
*
thandle
;
void
*
chandle
;
void
*
shandle
;
void
*
thandle
;
void
*
chandle
;
}
SRecvInfo
;
#pragma pack(push, 1)
typedef
struct
{
char
version
:
4
;
// RPC version
char
comp
:
4
;
// compression algorithm, 0:no compression 1:lz4
char
resflag
:
2
;
// reserved bits
char
spi
:
3
;
// security parameter index
char
encrypt
:
3
;
// encrypt algorithm, 0: no encryption
char
version
:
4
;
// RPC version
char
comp
:
4
;
// compression algorithm, 0:no compression 1:lz4
char
resflag
:
2
;
// reserved bits
char
spi
:
3
;
// security parameter index
char
encrypt
:
3
;
// encrypt algorithm, 0: no encryption
uint16_t
tranId
;
// transcation ID
uint32_t
linkUid
;
// for unique connection ID assigned by client
uint64_t
ahandle
;
// ahandle assigned by client
...
...
@@ -70,11 +75,10 @@ typedef struct {
}
SRpcDigest
;
#pragma pack(pop)
#endif
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_RPCHEAD_H
source/libs/transport/inc/rpcTcp.h
浏览文件 @
b9450007
...
...
@@ -15,11 +15,14 @@
#ifndef _rpc_tcp_header_
#define _rpc_tcp_header_
#include <stdint.h>
#ifdef __cplusplus
extern
"C"
{
#endif
#ifdef USE_UV
#else
void
*
taosInitTcpServer
(
uint32_t
ip
,
uint16_t
port
,
char
*
label
,
int
numOfThreads
,
void
*
fp
,
void
*
shandle
);
void
taosStopTcpServer
(
void
*
param
);
void
taosCleanUpTcpServer
(
void
*
param
);
...
...
@@ -32,6 +35,8 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
void
taosCloseTcpConnection
(
void
*
chandle
);
int
taosSendTcpData
(
uint32_t
ip
,
uint16_t
port
,
void
*
data
,
int
len
,
void
*
chandle
);
#endif
#ifdef __cplusplus
}
#endif
...
...
source/libs/transport/inc/transportInt.h
浏览文件 @
b9450007
...
...
@@ -20,6 +20,11 @@
extern
"C"
{
#endif
#ifdef USE_UV
#else
#endif
#ifdef __cplusplus
}
#endif
...
...
source/libs/transport/src/rpcCache.c
浏览文件 @
b9450007
...
...
@@ -13,27 +13,30 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "rpcCache.h"
#include "os.h"
#include "rpcLog.h"
#include "taosdef.h"
#include "tglobal.h"
#include "tmempool.h"
#include "ttimer.h"
#include "tutil.h"
#include "rpcLog.h"
#include "rpcCache.h"
#ifdef USE_UV
#else
typedef
struct
SConnHash
{
char
fqdn
[
TSDB_FQDN_LEN
];
uint16_t
port
;
char
connType
;
struct
SConnHash
*
prev
;
struct
SConnHash
*
next
;
void
*
data
;
void
*
data
;
uint64_t
time
;
}
SConnHash
;
typedef
struct
{
SConnHash
**
connHashList
;
SConnHash
**
connHashList
;
mpool_h
connHashMemPool
;
int
maxSessions
;
int
total
;
...
...
@@ -41,8 +44,8 @@ typedef struct {
int64_t
keepTimer
;
pthread_mutex_t
mutex
;
void
(
*
cleanFp
)(
void
*
);
void
*
tmrCtrl
;
void
*
pTimer
;
void
*
tmrCtrl
;
void
*
pTimer
;
int64_t
*
lockedBy
;
}
SConnCache
;
...
...
@@ -134,7 +137,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in
pNode
->
prev
=
NULL
;
pNode
->
time
=
time
;
rpcLockCache
(
pCache
->
lockedBy
+
hash
);
rpcLockCache
(
pCache
->
lockedBy
+
hash
);
pNode
->
next
=
pCache
->
connHashList
[
hash
];
if
(
pCache
->
connHashList
[
hash
]
!=
NULL
)
(
pCache
->
connHashList
[
hash
])
->
prev
=
pNode
;
...
...
@@ -143,10 +146,11 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in
pCache
->
count
[
hash
]
++
;
rpcRemoveExpiredNodes
(
pCache
,
pNode
->
next
,
hash
,
time
);
rpcUnlockCache
(
pCache
->
lockedBy
+
hash
);
rpcUnlockCache
(
pCache
->
lockedBy
+
hash
);
pCache
->
total
++
;
// tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]);
// tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode,
// pCache->count[hash]);
return
;
}
...
...
@@ -163,7 +167,7 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy
uint64_t
time
=
taosGetTimestampMs
();
hash
=
rpcHashConn
(
pCache
,
fqdn
,
port
,
connType
);
rpcLockCache
(
pCache
->
lockedBy
+
hash
);
rpcLockCache
(
pCache
->
lockedBy
+
hash
);
pNode
=
pCache
->
connHashList
[
hash
];
while
(
pNode
)
{
...
...
@@ -197,12 +201,14 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy
pCache
->
count
[
hash
]
--
;
}
rpcUnlockCache
(
pCache
->
lockedBy
+
hash
);
rpcUnlockCache
(
pCache
->
lockedBy
+
hash
);
if
(
pData
)
{
//tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]);
// tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode,
// pCache->count[hash]);
}
else
{
//tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, pCache->count[hash]);
// tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash,
// pCache->count[hash]);
}
return
pData
;
...
...
@@ -221,10 +227,10 @@ static void rpcCleanConnCache(void *handle, void *tmrId) {
uint64_t
time
=
taosGetTimestampMs
();
for
(
hash
=
0
;
hash
<
pCache
->
maxSessions
;
++
hash
)
{
rpcLockCache
(
pCache
->
lockedBy
+
hash
);
rpcLockCache
(
pCache
->
lockedBy
+
hash
);
pNode
=
pCache
->
connHashList
[
hash
];
rpcRemoveExpiredNodes
(
pCache
,
pNode
,
hash
,
time
);
rpcUnlockCache
(
pCache
->
lockedBy
+
hash
);
rpcUnlockCache
(
pCache
->
lockedBy
+
hash
);
}
// tTrace("timer, total connections in cache:%d", pCache->total);
...
...
@@ -233,7 +239,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) {
}
static
void
rpcRemoveExpiredNodes
(
SConnCache
*
pCache
,
SConnHash
*
pNode
,
int
hash
,
uint64_t
time
)
{
if
(
pNode
==
NULL
||
(
time
<
pCache
->
keepTimer
+
pNode
->
time
)
)
return
;
if
(
pNode
==
NULL
||
(
time
<
pCache
->
keepTimer
+
pNode
->
time
))
return
;
SConnHash
*
pPrev
=
pNode
->
prev
,
*
pNext
;
...
...
@@ -242,7 +248,8 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash
pNext
=
pNode
->
next
;
pCache
->
total
--
;
pCache
->
count
[
hash
]
--
;
//tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode,
// tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port,
// pNode->connType, hash, pNode,
// pCache->count[hash]);
taosMemPoolFree
(
pCache
->
connHashMemPool
,
(
char
*
)
pNode
);
pNode
=
pNext
;
...
...
@@ -257,7 +264,7 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash
static
int
rpcHashConn
(
void
*
handle
,
char
*
fqdn
,
uint16_t
port
,
int8_t
connType
)
{
SConnCache
*
pCache
=
(
SConnCache
*
)
handle
;
int
hash
=
0
;
char
*
temp
=
fqdn
;
char
*
temp
=
fqdn
;
while
(
*
temp
)
{
hash
+=
*
temp
;
...
...
@@ -288,4 +295,4 @@ static void rpcUnlockCache(int64_t *lockedBy) {
assert
(
false
);
}
}
#endif
source/libs/transport/src/rpcMain.c
浏览文件 @
b9450007
...
...
@@ -13,27 +13,175 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <uv.h>
#include "lz4.h"
#include "os.h"
#include "rpcCache.h"
#include "rpcHead.h"
#include "rpcLog.h"
#include "rpcTcp.h"
#include "rpcUdp.h"
#include "taoserror.h"
#include "tglobal.h"
#include "thash.h"
#include "tidpool.h"
#include "tmd5.h"
#include "tmempool.h"
#include "ttimer.h"
#include "tutil.h"
#include "lz4.h"
#include "tref.h"
#include "taoserror.h"
#include "tglobal.h"
#include "tmsg.h"
#include "tref.h"
#include "trpc.h"
#include "thash.h"
#include "rpcLog.h"
#include "rpcUdp.h"
#include "rpcCache.h"
#include "rpcTcp.h"
#include "rpcHead.h"
#include "ttimer.h"
#include "tutil.h"
#ifdef USE_UV
#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member)))
int32_t
rpcInit
()
{
return
-
1
;
}
void
rpcCleanup
()
{
return
;
};
void
*
rpcOpen
(
const
SRpcInit
*
pRpc
)
{
return
NULL
;
}
void
rpcClose
(
void
*
arg
)
{
return
;
}
void
*
rpcMallocCont
(
int
contLen
)
{
return
NULL
;
}
void
rpcFreeCont
(
void
*
cont
)
{
return
;
}
void
*
rpcReallocCont
(
void
*
ptr
,
int
contLen
)
{
return
NULL
;
}
void
rpcSendRequest
(
void
*
thandle
,
const
SEpSet
*
pEpSet
,
SRpcMsg
*
pMsg
,
int64_t
*
rid
)
{
return
;
}
void
rpcSendResponse
(
const
SRpcMsg
*
pMsg
)
{}
void
rpcSendRedirectRsp
(
void
*
pConn
,
const
SEpSet
*
pEpSet
)
{}
int
rpcGetConnInfo
(
void
*
thandle
,
SRpcConnInfo
*
pInfo
)
{
return
-
1
;
}
void
rpcSendRecv
(
void
*
shandle
,
SEpSet
*
pEpSet
,
SRpcMsg
*
pReq
,
SRpcMsg
*
pRsp
)
{
return
;
}
int
rpcReportProgress
(
void
*
pConn
,
char
*
pCont
,
int
contLen
)
{
return
-
1
;
}
void
rpcCancelRequest
(
int64_t
rid
)
{
return
;
}
typedef
struct
SThreadObj
{
pthread_t
thread
;
uv_pipe_t
*
pipe
;
uv_loop_t
*
loop
;
uv_async_t
*
workerAsync
;
//
int
fd
;
}
SThreadObj
;
typedef
struct
SServerObj
{
uv_tcp_t
server
;
uv_loop_t
*
loop
;
int
workerIdx
;
int
numOfThread
;
SThreadObj
**
pThreadObj
;
uv_pipe_t
**
pipe
;
}
SServerObj
;
typedef
struct
SConnCtx
{
uv_tcp_t
*
pClient
;
uv_timer_t
*
pTimer
;
uv_async_t
*
pWorkerAsync
;
int
ref
;
}
SConnCtx
;
void
allocBuffer
(
uv_handle_t
*
handle
,
size_t
suggested_size
,
uv_buf_t
*
buf
)
{
buf
->
base
=
malloc
(
suggested_size
);
buf
->
len
=
suggested_size
;
}
void
onTimeout
(
uv_timer_t
*
handle
)
{
// opt
tDebug
(
"time out"
);
}
void
onRead
(
uv_stream_t
*
cli
,
ssize_t
nread
,
const
uv_buf_t
*
buf
)
{
// opt
tDebug
(
"data already was read on a stream"
);
}
void
onWrite
(
uv_write_t
*
req
,
int
status
)
{
// opt
if
(
req
)
tDebug
(
"data already was written on stream"
);
}
static
void
workerAsyncCB
(
uv_async_t
*
handle
)
{
// opt
SThreadObj
*
pObj
=
container_of
(
handle
,
SThreadObj
,
workerAsync
);
}
void
onAccept
(
uv_stream_t
*
stream
,
int
status
)
{
if
(
status
==
-
1
)
{
return
;
}
SServerObj
*
pObj
=
container_of
(
stream
,
SServerObj
,
server
);
tDebug
(
"new conntion accepted by main server, dispatch to one worker thread"
);
uv_tcp_t
*
cli
=
(
uv_tcp_t
*
)
malloc
(
sizeof
(
uv_tcp_t
));
uv_tcp_init
(
pObj
->
loop
,
cli
);
if
(
uv_accept
(
stream
,
(
uv_stream_t
*
)
cli
)
==
0
)
{
uv_write_t
*
wr
=
(
uv_write_t
*
)
malloc
(
sizeof
(
uv_write_t
));
uv_buf_t
buf
=
uv_buf_init
(
"a"
,
1
);
// despatch to worker thread
pObj
->
workerIdx
=
(
pObj
->
workerIdx
+
1
)
%
pObj
->
numOfThread
;
uv_write2
(
wr
,
(
uv_stream_t
*
)
&
(
pObj
->
pipe
[
pObj
->
workerIdx
][
0
]),
&
buf
,
1
,
(
uv_stream_t
*
)
cli
,
onWrite
);
}
else
{
uv_close
((
uv_handle_t
*
)
cli
,
NULL
);
}
}
void
onConnection
(
uv_stream_t
*
q
,
ssize_t
nread
,
const
uv_buf_t
*
buf
)
{
if
(
nread
<
0
)
{
if
(
nread
!=
UV_EOF
)
{
tError
(
"read error %s"
,
uv_err_name
(
nread
));
}
// TODO(log other failure reason)
uv_close
((
uv_handle_t
*
)
q
,
NULL
);
return
;
}
SThreadObj
*
pObj
=
(
SThreadObj
*
)
container_of
(
q
,
struct
SThreadObj
,
pipe
);
uv_pipe_t
*
pipe
=
(
uv_pipe_t
*
)
q
;
if
(
!
uv_pipe_pending_count
(
pipe
))
{
tError
(
"No pending count"
);
return
;
}
uv_handle_type
pending
=
uv_pipe_pending_type
(
pipe
);
assert
(
pending
==
UV_TCP
);
SConnCtx
*
pConn
=
malloc
(
sizeof
(
SConnCtx
));
/* init conn timer*/
pConn
->
pTimer
=
malloc
(
sizeof
(
uv_timer_t
));
uv_timer_init
(
pObj
->
loop
,
pConn
->
pTimer
);
pConn
->
pClient
=
(
uv_tcp_t
*
)
malloc
(
sizeof
(
uv_tcp_t
));
pConn
->
pWorkerAsync
=
pObj
->
workerAsync
;
// thread safty
uv_tcp_init
(
pObj
->
loop
,
pConn
->
pClient
);
if
(
uv_accept
(
q
,
(
uv_stream_t
*
)(
pConn
->
pClient
))
==
0
)
{
uv_os_fd_t
fd
;
uv_fileno
((
const
uv_handle_t
*
)
pConn
->
pClient
,
&
fd
);
tDebug
(
"new connection created: %d"
,
fd
);
uv_timer_start
(
pConn
->
pTimer
,
onTimeout
,
10
,
0
);
uv_read_start
((
uv_stream_t
*
)(
pConn
->
pClient
),
allocBuffer
,
onRead
);
}
else
{
uv_timer_stop
(
pConn
->
pTimer
);
free
(
pConn
->
pTimer
);
uv_close
((
uv_handle_t
*
)
pConn
->
pClient
,
NULL
);
free
(
pConn
->
pClient
);
free
(
pConn
);
}
}
void
*
workerThread
(
void
*
arg
)
{
SThreadObj
*
pObj
=
(
SThreadObj
*
)
arg
;
int
fd
=
pObj
->
fd
;
pObj
->
loop
=
(
uv_loop_t
*
)
malloc
(
sizeof
(
uv_loop_t
));
uv_loop_init
(
pObj
->
loop
);
uv_pipe_init
(
pObj
->
loop
,
pObj
->
pipe
,
1
);
uv_pipe_open
(
pObj
->
pipe
,
fd
);
pObj
->
workerAsync
=
malloc
(
sizeof
(
uv_async_t
));
uv_async_init
(
pObj
->
loop
,
pObj
->
workerAsync
,
workerAsyncCB
);
uv_read_start
((
uv_stream_t
*
)
pObj
->
pipe
,
allocBuffer
,
onConnection
);
}
#else
#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest))
#define rpcHeadFromCont(cont) ((SRpcHead
*)
((char*)cont - sizeof(SRpcHead)))
#define rpcHeadFromCont(cont) ((SRpcHead
*)
((char*)cont - sizeof(SRpcHead)))
#define rpcContFromHead(msg) (msg + sizeof(SRpcHead))
#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead))
#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead))
...
...
@@ -58,24 +206,24 @@ typedef struct {
int
(
*
afp
)(
void
*
parent
,
char
*
user
,
char
*
spi
,
char
*
encrypt
,
char
*
secret
,
char
*
ckey
);
int32_t
refCount
;
void
*
parent
;
void
*
idPool
;
// handle to ID pool
void
*
tmrCtrl
;
// handle to timer
SHashObj
*
hash
;
// handle returned by hash utility
void
*
tcphandle
;
// returned handle from TCP initialization
void
*
udphandle
;
// returned handle from UDP initialization
void
*
pCache
;
// connection cache
void
*
parent
;
void
*
idPool
;
// handle to ID pool
void
*
tmrCtrl
;
// handle to timer
SHashObj
*
hash
;
// handle returned by hash utility
void
*
tcphandle
;
// returned handle from TCP initialization
void
*
udphandle
;
// returned handle from UDP initialization
void
*
pCache
;
// connection cache
pthread_mutex_t
mutex
;
struct
SRpcConn
*
connList
;
// connection list
}
SRpcInfo
;
typedef
struct
{
SRpcInfo
*
pRpc
;
// associated SRpcInfo
SRpcInfo
*
pRpc
;
// associated SRpcInfo
SEpSet
epSet
;
// ip list provided by app
void
*
ahandle
;
// handle provided by app
void
*
ahandle
;
// handle provided by app
struct
SRpcConn
*
pConn
;
// pConn allocated
tmsg_t
msgType
;
// message type
uint8_t
*
pCont
;
// content provided by app
uint8_t
*
pCont
;
// content provided by app
int32_t
contLen
;
// content length
int32_t
code
;
// error code
int16_t
numOfTry
;
// number of try for different servers
...
...
@@ -83,14 +231,14 @@ typedef struct {
int8_t
redirect
;
// flag to indicate redirect
int8_t
connType
;
// connection type
int64_t
rid
;
// refId returned by taosAddRef
SRpcMsg
*
pRsp
;
// for synchronous API
tsem_t
*
pSem
;
// for synchronous API
SEpSet
*
pSet
;
// for synchronous API
SRpcMsg
*
pRsp
;
// for synchronous API
tsem_t
*
pSem
;
// for synchronous API
SEpSet
*
pSet
;
// for synchronous API
char
msg
[
0
];
// RpcHead starts from here
}
SRpcReqContext
;
typedef
struct
SRpcConn
{
char
info
[
48
];
// debug info: label + pConn + ahandle
char
info
[
48
];
// debug info: label + pConn + ahandle
int
sid
;
// session ID
uint32_t
ownId
;
// own link ID
uint32_t
peerId
;
// peer link ID
...
...
@@ -110,17 +258,17 @@ typedef struct SRpcConn {
uint16_t
inTranId
;
// transcation ID for incoming msg
tmsg_t
outType
;
// message type for outgoing request
tmsg_t
inType
;
// message type for incoming request
void
*
chandle
;
// handle passed by TCP/UDP connection layer
void
*
ahandle
;
// handle provided by upper app layter
void
*
chandle
;
// handle passed by TCP/UDP connection layer
void
*
ahandle
;
// handle provided by upper app layter
int
retry
;
// number of retry for sending request
int
tretry
;
// total retry
void
*
pTimer
;
// retry timer to monitor the response
void
*
pIdleTimer
;
// idle timer
char
*
pRspMsg
;
// response message including header
void
*
pTimer
;
// retry timer to monitor the response
void
*
pIdleTimer
;
// idle timer
char
*
pRspMsg
;
// response message including header
int
rspMsgLen
;
// response messag length
char
*
pReqMsg
;
// request message including header
char
*
pReqMsg
;
// request message including header
int
reqMsgLen
;
// request message length
SRpcInfo
*
pRpc
;
// the associated SRpcInfo
SRpcInfo
*
pRpc
;
// the associated SRpcInfo
int8_t
connType
;
// connection type
int64_t
lockedBy
;
// lock for connection
SRpcReqContext
*
pContext
;
// request context
...
...
@@ -137,7 +285,7 @@ int tsRpcOverhead;
static
int
tsRpcRefId
=
-
1
;
static
int32_t
tsRpcNum
=
0
;
//static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
//
static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
// server:0 client:1 tcp:2 udp:0
#define RPC_CONN_UDPS 0
...
...
@@ -146,18 +294,10 @@ static int32_t tsRpcNum = 0;
#define RPC_CONN_TCPC 3
void
*
(
*
taosInitConn
[])(
uint32_t
ip
,
uint16_t
port
,
char
*
label
,
int
threads
,
void
*
fp
,
void
*
shandle
)
=
{
taosInitUdpConnection
,
taosInitUdpConnection
,
taosInitTcpServer
,
taosInitTcpClient
};
taosInitUdpConnection
,
taosInitUdpConnection
,
taosInitTcpServer
,
taosInitTcpClient
};
void
(
*
taosCleanUpConn
[])(
void
*
thandle
)
=
{
taosCleanUpUdpConnection
,
taosCleanUpUdpConnection
,
taosCleanUpTcpServer
,
taosCleanUpTcpClient
};
void
(
*
taosCleanUpConn
[])(
void
*
thandle
)
=
{
taosCleanUpUdpConnection
,
taosCleanUpUdpConnection
,
taosCleanUpTcpServer
,
taosCleanUpTcpClient
};
void
(
*
taosStopConn
[])(
void
*
thandle
)
=
{
taosStopUdpConnection
,
...
...
@@ -167,11 +307,7 @@ void (*taosStopConn[])(void *thandle) = {
};
int
(
*
taosSendData
[])(
uint32_t
ip
,
uint16_t
port
,
void
*
data
,
int
len
,
void
*
chandle
)
=
{
taosSendUdpData
,
taosSendUdpData
,
taosSendTcpData
,
taosSendTcpData
};
taosSendUdpData
,
taosSendUdpData
,
taosSendTcpData
,
taosSendTcpData
};
void
*
(
*
taosOpenConn
[])(
void
*
shandle
,
void
*
thandle
,
uint32_t
ip
,
uint16_t
port
)
=
{
taosOpenUdpConnection
,
...
...
@@ -180,12 +316,7 @@ void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port
taosOpenTcpClientConnection
,
};
void
(
*
taosCloseConn
[])(
void
*
chandle
)
=
{
NULL
,
NULL
,
taosCloseTcpConnection
,
taosCloseTcpConnection
};
void
(
*
taosCloseConn
[])(
void
*
chandle
)
=
{
NULL
,
NULL
,
taosCloseTcpConnection
,
taosCloseTcpConnection
};
static
SRpcConn
*
rpcOpenConn
(
SRpcInfo
*
pRpc
,
char
*
peerFqdn
,
uint16_t
peerPort
,
int8_t
connType
);
static
void
rpcCloseConn
(
void
*
thandle
);
...
...
@@ -208,7 +339,7 @@ static void rpcProcessIdleTimer(void *param, void *tmrId);
static
void
rpcProcessProgressTimer
(
void
*
param
,
void
*
tmrId
);
static
void
rpcFreeMsg
(
void
*
msg
);
static
int32_t
rpcCompressRpcMsg
(
char
*
pCont
,
int32_t
contLen
);
static
int32_t
rpcCompressRpcMsg
(
char
*
pCont
,
int32_t
contLen
);
static
SRpcHead
*
rpcDecompressRpcMsg
(
SRpcHead
*
pHead
);
static
int
rpcAddAuthPart
(
SRpcConn
*
pConn
,
char
*
msg
,
int
msgLen
);
static
int
rpcCheckAuthentication
(
SRpcConn
*
pConn
,
char
*
msg
,
int
msgLen
);
...
...
@@ -244,22 +375,22 @@ void rpcCleanup(void) {
void
*
rpcOpen
(
const
SRpcInit
*
pInit
)
{
SRpcInfo
*
pRpc
;
//pthread_once(&tsRpcInit, rpcInit);
//
pthread_once(&tsRpcInit, rpcInit);
pRpc
=
(
SRpcInfo
*
)
calloc
(
1
,
sizeof
(
SRpcInfo
));
if
(
pRpc
==
NULL
)
return
NULL
;
if
(
pInit
->
label
)
tstrncpy
(
pRpc
->
label
,
pInit
->
label
,
sizeof
(
pRpc
->
label
));
if
(
pInit
->
label
)
tstrncpy
(
pRpc
->
label
,
pInit
->
label
,
sizeof
(
pRpc
->
label
));
pRpc
->
connType
=
pInit
->
connType
;
if
(
pRpc
->
connType
==
TAOS_CONN_CLIENT
)
{
pRpc
->
numOfThreads
=
pInit
->
numOfThreads
;
}
else
{
pRpc
->
numOfThreads
=
pInit
->
numOfThreads
>
TSDB_MAX_RPC_THREADS
?
TSDB_MAX_RPC_THREADS
:
pInit
->
numOfThreads
;
pRpc
->
numOfThreads
=
pInit
->
numOfThreads
>
TSDB_MAX_RPC_THREADS
?
TSDB_MAX_RPC_THREADS
:
pInit
->
numOfThreads
;
}
pRpc
->
idleTime
=
pInit
->
idleTime
;
pRpc
->
localPort
=
pInit
->
localPort
;
pRpc
->
afp
=
pInit
->
afp
;
pRpc
->
sessions
=
pInit
->
sessions
+
1
;
pRpc
->
sessions
=
pInit
->
sessions
+
1
;
if
(
pInit
->
user
)
tstrncpy
(
pRpc
->
user
,
pInit
->
user
,
sizeof
(
pRpc
->
user
));
if
(
pInit
->
secret
)
memcpy
(
pRpc
->
secret
,
pInit
->
secret
,
sizeof
(
pRpc
->
secret
));
if
(
pInit
->
ckey
)
tstrncpy
(
pRpc
->
ckey
,
pInit
->
ckey
,
sizeof
(
pRpc
->
ckey
));
...
...
@@ -279,14 +410,14 @@ void *rpcOpen(const SRpcInit *pInit) {
return
NULL
;
}
pRpc
->
idPool
=
taosInitIdPool
(
pRpc
->
sessions
-
1
);
pRpc
->
idPool
=
taosInitIdPool
(
pRpc
->
sessions
-
1
);
if
(
pRpc
->
idPool
==
NULL
)
{
tError
(
"%s failed to init ID pool"
,
pRpc
->
label
);
rpcClose
(
pRpc
);
return
NULL
;
}
pRpc
->
tmrCtrl
=
taosTmrInit
(
pRpc
->
sessions
*
2
+
1
,
50
,
10000
,
pRpc
->
label
);
pRpc
->
tmrCtrl
=
taosTmrInit
(
pRpc
->
sessions
*
2
+
1
,
50
,
10000
,
pRpc
->
label
);
if
(
pRpc
->
tmrCtrl
==
NULL
)
{
tError
(
"%s failed to init timers"
,
pRpc
->
label
);
rpcClose
(
pRpc
);
...
...
@@ -302,7 +433,7 @@ void *rpcOpen(const SRpcInit *pInit) {
}
}
else
{
pRpc
->
pCache
=
rpcOpenConnCache
(
pRpc
->
sessions
,
rpcCloseConn
,
pRpc
->
tmrCtrl
,
pRpc
->
idleTime
*
20
);
if
(
pRpc
->
pCache
==
NULL
)
{
if
(
pRpc
->
pCache
==
NULL
)
{
tError
(
"%s failed to init connection cache"
,
pRpc
->
label
);
rpcClose
(
pRpc
);
return
NULL
;
...
...
@@ -311,10 +442,10 @@ void *rpcOpen(const SRpcInit *pInit) {
pthread_mutex_init
(
&
pRpc
->
mutex
,
NULL
);
pRpc
->
tcphandle
=
(
*
taosInitConn
[
pRpc
->
connType
|
RPC_CONN_TCP
])(
0
,
pRpc
->
localPort
,
pRpc
->
label
,
pRpc
->
numOfThreads
,
rpcProcessMsgFromPeer
,
pRpc
);
pRpc
->
udphandle
=
(
*
taosInitConn
[
pRpc
->
connType
])(
0
,
pRpc
->
localPort
,
pRpc
->
label
,
pRpc
->
numOfThreads
,
rpcProcessMsgFromPeer
,
pRpc
);
pRpc
->
tcphandle
=
(
*
taosInitConn
[
pRpc
->
connType
|
RPC_CONN_TCP
])(
0
,
pRpc
->
localPort
,
pRpc
->
label
,
pRpc
->
numOfThreads
,
rpcProcessMsgFromPeer
,
pRpc
);
pRpc
->
udphandle
=
(
*
taosInitConn
[
pRpc
->
connType
])(
0
,
pRpc
->
localPort
,
pRpc
->
label
,
pRpc
->
numOfThreads
,
rpcProcessMsgFromPeer
,
pRpc
);
if
(
pRpc
->
tcphandle
==
NULL
||
pRpc
->
udphandle
==
NULL
)
{
tError
(
"%s failed to init network, port:%d"
,
pRpc
->
label
,
pRpc
->
localPort
);
...
...
@@ -375,7 +506,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
if
(
ptr
==
NULL
)
return
rpcMallocCont
(
contLen
);
char
*
start
=
((
char
*
)
ptr
)
-
sizeof
(
SRpcReqContext
)
-
sizeof
(
SRpcHead
);
if
(
contLen
==
0
)
{
if
(
contLen
==
0
)
{
free
(
start
);
return
NULL
;
}
...
...
@@ -391,11 +522,11 @@ void *rpcReallocCont(void *ptr, int contLen) {
}
void
rpcSendRequest
(
void
*
shandle
,
const
SEpSet
*
pEpSet
,
SRpcMsg
*
pMsg
,
int64_t
*
pRid
)
{
SRpcInfo
*
pRpc
=
(
SRpcInfo
*
)
shandle
;
SRpcInfo
*
pRpc
=
(
SRpcInfo
*
)
shandle
;
SRpcReqContext
*
pContext
;
int
contLen
=
rpcCompressRpcMsg
(
pMsg
->
pCont
,
pMsg
->
contLen
);
pContext
=
(
SRpcReqContext
*
)
((
char
*
)
pMsg
->
pCont
-
sizeof
(
SRpcHead
)
-
sizeof
(
SRpcReqContext
));
pContext
=
(
SRpcReqContext
*
)
((
char
*
)
pMsg
->
pCont
-
sizeof
(
SRpcHead
)
-
sizeof
(
SRpcReqContext
));
pContext
->
ahandle
=
pMsg
->
ahandle
;
pContext
->
pRpc
=
(
SRpcInfo
*
)
shandle
;
pContext
->
epSet
=
*
pEpSet
;
...
...
@@ -405,15 +536,14 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t
pContext
->
oldInUse
=
pEpSet
->
inUse
;
pContext
->
connType
=
RPC_CONN_UDPC
;
if
(
contLen
>
tsRpcMaxUdpSize
||
tsRpcForceTcp
)
pContext
->
connType
=
RPC_CONN_TCPC
;
if
(
contLen
>
tsRpcMaxUdpSize
||
tsRpcForceTcp
)
pContext
->
connType
=
RPC_CONN_TCPC
;
// connection type is application specific.
// for TDengine, all the query, show commands shall have TCP connection
tmsg_t
type
=
pMsg
->
msgType
;
if
(
type
==
TDMT_VND_QUERY
||
type
==
TDMT_MND_SHOW_RETRIEVE
||
type
==
TDMT_VND_FETCH
||
type
==
TDMT_MND_VGROUP_LIST
||
type
==
TDMT_VND_TABLES_META
||
type
==
TDMT_VND_TABLE_META
||
type
==
TDMT_MND_SHOW
||
type
==
TDMT_MND_STATUS
||
type
==
TDMT_VND_ALTER_TABLE
)
if
(
type
==
TDMT_VND_QUERY
||
type
==
TDMT_MND_SHOW_RETRIEVE
||
type
==
TDMT_VND_FETCH
||
type
==
TDMT_MND_VGROUP_LIST
||
type
==
TDMT_VND_TABLES_META
||
type
==
TDMT_VND_TABLE_META
||
type
==
TDMT_MND_SHOW
||
type
==
TDMT_MND_STATUS
||
type
==
TDMT_VND_ALTER_TABLE
)
pContext
->
connType
=
RPC_CONN_TCPC
;
pContext
->
rid
=
taosAddRef
(
tsRpcRefId
,
pContext
);
...
...
@@ -428,23 +558,23 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
int
msgLen
=
0
;
SRpcConn
*
pConn
=
(
SRpcConn
*
)
pRsp
->
handle
;
SRpcMsg
rpcMsg
=
*
pRsp
;
SRpcMsg
*
pMsg
=
&
rpcMsg
;
SRpcMsg
*
pMsg
=
&
rpcMsg
;
SRpcInfo
*
pRpc
=
pConn
->
pRpc
;
if
(
pMsg
->
pCont
==
NULL
)
{
if
(
pMsg
->
pCont
==
NULL
)
{
pMsg
->
pCont
=
rpcMallocCont
(
0
);
pMsg
->
contLen
=
0
;
}
SRpcHead
*
pHead
=
rpcHeadFromCont
(
pMsg
->
pCont
);
char
*
msg
=
(
char
*
)
pHead
;
char
*
msg
=
(
char
*
)
pHead
;
pMsg
->
contLen
=
rpcCompressRpcMsg
(
pMsg
->
pCont
,
pMsg
->
contLen
);
msgLen
=
rpcMsgLenFromCont
(
pMsg
->
contLen
);
rpcLockConn
(
pConn
);
if
(
pConn
->
inType
==
0
||
pConn
->
user
[
0
]
==
0
)
{
if
(
pConn
->
inType
==
0
||
pConn
->
user
[
0
]
==
0
)
{
tError
(
"%s, connection is already released, rsp wont be sent"
,
pConn
->
info
);
rpcUnlockConn
(
pConn
);
rpcFreeCont
(
pMsg
->
pCont
);
...
...
@@ -454,7 +584,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
// set msg header
pHead
->
version
=
1
;
pHead
->
msgType
=
pConn
->
inType
+
1
;
pHead
->
msgType
=
pConn
->
inType
+
1
;
pHead
->
spi
=
pConn
->
spi
;
pHead
->
encrypt
=
pConn
->
encrypt
;
pHead
->
tranId
=
pConn
->
inTranId
;
...
...
@@ -463,7 +593,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
pHead
->
linkUid
=
pConn
->
linkUid
;
pHead
->
port
=
htons
(
pConn
->
localPort
);
pHead
->
code
=
htonl
(
pMsg
->
code
);
pHead
->
ahandle
=
(
uint64_t
)
pConn
->
ahandle
;
pHead
->
ahandle
=
(
uint64_t
)
pConn
->
ahandle
;
// set pConn parameters
pConn
->
inType
=
0
;
...
...
@@ -482,8 +612,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
rpcSendMsgToPeer
(
pConn
,
msg
,
msgLen
);
// if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured
if
(
pConn
->
secured
==
0
&&
pMsg
->
code
!=
TSDB_CODE_RPC_NOT_READY
)
pConn
->
secured
=
1
;
// connection shall be secured
if
(
pConn
->
secured
==
0
&&
pMsg
->
code
!=
TSDB_CODE_RPC_NOT_READY
)
pConn
->
secured
=
1
;
// connection shall be secured
if
(
pConn
->
pReqMsg
)
rpcFreeCont
(
pConn
->
pReqMsg
);
pConn
->
pReqMsg
=
NULL
;
...
...
@@ -527,7 +656,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
void
rpcSendRecv
(
void
*
shandle
,
SEpSet
*
pEpSet
,
SRpcMsg
*
pMsg
,
SRpcMsg
*
pRsp
)
{
SRpcReqContext
*
pContext
;
pContext
=
(
SRpcReqContext
*
)
((
char
*
)
pMsg
->
pCont
-
sizeof
(
SRpcHead
)
-
sizeof
(
SRpcReqContext
));
pContext
=
(
SRpcReqContext
*
)
((
char
*
)
pMsg
->
pCont
-
sizeof
(
SRpcHead
)
-
sizeof
(
SRpcReqContext
));
memset
(
pRsp
,
0
,
sizeof
(
SRpcMsg
));
...
...
@@ -567,7 +696,6 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) {
}
void
rpcCancelRequest
(
int64_t
rid
)
{
SRpcReqContext
*
pContext
=
taosAcquireRef
(
tsRpcRefId
,
rid
);
if
(
pContext
==
NULL
)
return
;
...
...
@@ -577,7 +705,7 @@ void rpcCancelRequest(int64_t rid) {
}
static
void
rpcFreeMsg
(
void
*
msg
)
{
if
(
msg
)
{
if
(
msg
)
{
char
*
temp
=
(
char
*
)
msg
-
sizeof
(
SRpcReqContext
);
free
(
temp
);
tTrace
(
"free mem: %p"
,
temp
);
...
...
@@ -604,7 +732,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort,
pConn
->
connType
=
connType
;
if
(
taosOpenConn
[
connType
])
{
void
*
shandle
=
(
connType
&
RPC_CONN_TCP
)
?
pRpc
->
tcphandle
:
pRpc
->
udphandle
;
void
*
shandle
=
(
connType
&
RPC_CONN_TCP
)
?
pRpc
->
tcphandle
:
pRpc
->
udphandle
;
pConn
->
chandle
=
(
*
taosOpenConn
[
connType
])(
shandle
,
pConn
,
pConn
->
peerIp
,
pConn
->
peerPort
);
if
(
pConn
->
chandle
==
NULL
)
{
tError
(
"failed to connect to:%s:%d"
,
taosIpStr
(
pConn
->
peerIp
),
pConn
->
peerPort
);
...
...
@@ -629,9 +757,10 @@ static void rpcReleaseConn(SRpcConn *pConn) {
taosTmrStopA
(
&
pConn
->
pTimer
);
taosTmrStopA
(
&
pConn
->
pIdleTimer
);
if
(
pRpc
->
connType
==
TAOS_CONN_SERVER
)
{
if
(
pRpc
->
connType
==
TAOS_CONN_SERVER
)
{
char
hashstr
[
40
]
=
{
0
};
size_t
size
=
snprintf
(
hashstr
,
sizeof
(
hashstr
),
"%x:%x:%x:%d"
,
pConn
->
peerIp
,
pConn
->
linkUid
,
pConn
->
peerId
,
pConn
->
connType
);
size_t
size
=
snprintf
(
hashstr
,
sizeof
(
hashstr
),
"%x:%x:%x:%d"
,
pConn
->
peerIp
,
pConn
->
linkUid
,
pConn
->
peerId
,
pConn
->
connType
);
taosHashRemove
(
pRpc
->
hash
,
hashstr
,
size
);
rpcFreeMsg
(
pConn
->
pRspMsg
);
// it may have a response msg saved, but not request msg
pConn
->
pRspMsg
=
NULL
;
...
...
@@ -682,8 +811,7 @@ static void rpcCloseConn(void *thandle) {
rpcLockConn
(
pConn
);
if
(
pConn
->
user
[
0
])
rpcReleaseConn
(
pConn
);
if
(
pConn
->
user
[
0
])
rpcReleaseConn
(
pConn
);
rpcUnlockConn
(
pConn
);
}
...
...
@@ -717,7 +845,8 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
char
hashstr
[
40
]
=
{
0
};
SRpcHead
*
pHead
=
(
SRpcHead
*
)
pRecv
->
msg
;
size_t
size
=
snprintf
(
hashstr
,
sizeof
(
hashstr
),
"%x:%x:%x:%d"
,
pRecv
->
ip
,
pHead
->
linkUid
,
pHead
->
sourceId
,
pRecv
->
connType
);
size_t
size
=
snprintf
(
hashstr
,
sizeof
(
hashstr
),
"%x:%x:%x:%d"
,
pRecv
->
ip
,
pHead
->
linkUid
,
pHead
->
sourceId
,
pRecv
->
connType
);
// check if it is already allocated
SRpcConn
**
ppConn
=
(
SRpcConn
**
)(
taosHashGet
(
pRpc
->
hash
,
hashstr
,
size
));
...
...
@@ -767,7 +896,8 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
}
taosHashPut
(
pRpc
->
hash
,
hashstr
,
size
,
(
char
*
)
&
pConn
,
POINTER_BYTES
);
tDebug
(
"%s %p server connection is allocated, uid:0x%x sid:%d key:%s"
,
pRpc
->
label
,
pConn
,
pConn
->
linkUid
,
sid
,
hashstr
);
tDebug
(
"%s %p server connection is allocated, uid:0x%x sid:%d key:%s"
,
pRpc
->
label
,
pConn
,
pConn
->
linkUid
,
sid
,
hashstr
);
}
return
pConn
;
...
...
@@ -805,10 +935,11 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) {
static
SRpcConn
*
rpcSetupConnToServer
(
SRpcReqContext
*
pContext
)
{
SRpcConn
*
pConn
;
SRpcInfo
*
pRpc
=
pContext
->
pRpc
;
SEpSet
*
pEpSet
=
&
pContext
->
epSet
;
SEpSet
*
pEpSet
=
&
pContext
->
epSet
;
pConn
=
rpcGetConnFromCache
(
pRpc
->
pCache
,
pEpSet
->
fqdn
[
pEpSet
->
inUse
],
pEpSet
->
port
[
pEpSet
->
inUse
],
pContext
->
connType
);
if
(
pConn
==
NULL
||
pConn
->
user
[
0
]
==
0
)
{
pConn
=
rpcGetConnFromCache
(
pRpc
->
pCache
,
pEpSet
->
fqdn
[
pEpSet
->
inUse
],
pEpSet
->
port
[
pEpSet
->
inUse
],
pContext
->
connType
);
if
(
pConn
==
NULL
||
pConn
->
user
[
0
]
==
0
)
{
pConn
=
rpcOpenConn
(
pRpc
,
pEpSet
->
fqdn
[
pEpSet
->
inUse
],
pEpSet
->
port
[
pEpSet
->
inUse
],
pContext
->
connType
);
}
...
...
@@ -825,13 +956,11 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) {
}
static
int
rpcProcessReqHead
(
SRpcConn
*
pConn
,
SRpcHead
*
pHead
)
{
if
(
pConn
->
peerId
==
0
)
{
pConn
->
peerId
=
pHead
->
sourceId
;
}
else
{
if
(
pConn
->
peerId
!=
pHead
->
sourceId
)
{
tDebug
(
"%s, source Id is changed, old:0x%08x new:0x%08x"
,
pConn
->
info
,
pConn
->
peerId
,
pHead
->
sourceId
);
tDebug
(
"%s, source Id is changed, old:0x%08x new:0x%08x"
,
pConn
->
info
,
pConn
->
peerId
,
pHead
->
sourceId
);
return
TSDB_CODE_RPC_INVALID_VALUE
;
}
}
...
...
@@ -856,8 +985,7 @@ static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) {
}
if
(
pConn
->
inType
!=
0
)
{
tDebug
(
"%s, last session is not finished, inTranId:%d tranId:%d"
,
pConn
->
info
,
pConn
->
inTranId
,
pHead
->
tranId
);
tDebug
(
"%s, last session is not finished, inTranId:%d tranId:%d"
,
pConn
->
info
,
pConn
->
inTranId
,
pHead
->
tranId
);
return
TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED
;
}
...
...
@@ -961,18 +1089,22 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont
if
(
TMSG_INDEX
(
pHead
->
msgType
)
>=
TDMT_MAX
||
TMSG_INDEX
(
pHead
->
msgType
)
<=
0
)
{
tDebug
(
"%s sid:%d, invalid message type:%d"
,
pRpc
->
label
,
sid
,
pHead
->
msgType
);
terrno
=
TSDB_CODE_RPC_INVALID_MSG_TYPE
;
return
NULL
;
terrno
=
TSDB_CODE_RPC_INVALID_MSG_TYPE
;
return
NULL
;
}
if
(
sid
<
0
||
sid
>=
pRpc
->
sessions
)
{
tDebug
(
"%s sid:%d, sid is out of range, max sid:%d, %s discarded"
,
pRpc
->
label
,
sid
,
pRpc
->
sessions
,
TMSG_INFO
(
pHead
->
msgType
));
terrno
=
TSDB_CODE_RPC_INVALID_SESSION_ID
;
return
NULL
;
tDebug
(
"%s sid:%d, sid is out of range, max sid:%d, %s discarded"
,
pRpc
->
label
,
sid
,
pRpc
->
sessions
,
TMSG_INFO
(
pHead
->
msgType
));
terrno
=
TSDB_CODE_RPC_INVALID_SESSION_ID
;
return
NULL
;
}
if
(
rpcIsReq
(
pHead
->
msgType
)
&&
htonl
(
pHead
->
msgVer
)
!=
tsVersion
>>
8
)
{
tDebug
(
"%s sid:%d, invalid client version:%x/%x %s"
,
pRpc
->
label
,
sid
,
htonl
(
pHead
->
msgVer
),
tsVersion
,
TMSG_INFO
(
pHead
->
msgType
));
terrno
=
TSDB_CODE_RPC_INVALID_VERSION
;
return
NULL
;
tDebug
(
"%s sid:%d, invalid client version:%x/%x %s"
,
pRpc
->
label
,
sid
,
htonl
(
pHead
->
msgVer
),
tsVersion
,
TMSG_INFO
(
pHead
->
msgType
));
terrno
=
TSDB_CODE_RPC_INVALID_VERSION
;
return
NULL
;
}
pConn
=
rpcGetConnObj
(
pRpc
,
sid
,
pRecv
);
...
...
@@ -1004,7 +1136,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont
// decrypt here
}
if
(
rpcIsReq
(
pHead
->
msgType
)
)
{
if
(
rpcIsReq
(
pHead
->
msgType
)
)
{
pConn
->
connType
=
pRecv
->
connType
;
terrno
=
rpcProcessReqHead
(
pConn
,
pHead
);
...
...
@@ -1013,7 +1145,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont
// client shall send the request within tsRpcTime again for UDP, double it
if
(
pConn
->
connType
!=
RPC_CONN_TCPS
)
pConn
->
pIdleTimer
=
taosTmrStart
(
rpcProcessIdleTimer
,
tsRpcTimer
*
2
,
pConn
,
pRpc
->
tmrCtrl
);
pConn
->
pIdleTimer
=
taosTmrStart
(
rpcProcessIdleTimer
,
tsRpcTimer
*
2
,
pConn
,
pRpc
->
tmrCtrl
);
}
else
{
terrno
=
rpcProcessRspHead
(
pConn
,
pHead
);
*
ppContext
=
pConn
->
pContext
;
...
...
@@ -1026,7 +1158,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont
}
static
void
doRpcReportBrokenLinkToServer
(
void
*
param
,
void
*
id
)
{
SRpcMsg
*
pRpcMsg
=
(
SRpcMsg
*
)(
param
);
SRpcMsg
*
pRpcMsg
=
(
SRpcMsg
*
)(
param
);
SRpcConn
*
pConn
=
(
SRpcConn
*
)(
pRpcMsg
->
handle
);
SRpcInfo
*
pRpc
=
pConn
->
pRpc
;
(
*
(
pRpc
->
cfp
))(
pRpc
->
parent
,
pRpcMsg
,
NULL
);
...
...
@@ -1100,13 +1232,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
taosIpPort2String
(
pRecv
->
ip
,
pRecv
->
port
,
ipstr
);
if
(
TMSG_INDEX
(
pHead
->
msgType
)
>=
1
&&
TMSG_INDEX
(
pHead
->
msgType
)
<
TDMT_MAX
)
{
tDebug
(
"%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x"
,
pRpc
->
label
,
pConn
,
(
void
*
)
pHead
->
ahandle
,
TMSG_INFO
(
pHead
->
msgType
),
ipstr
,
terrno
,
pRecv
->
msgLen
,
pHead
->
sourceId
,
pHead
->
destId
,
pHead
->
tranId
,
pHead
->
code
);
tDebug
(
"%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x"
,
pRpc
->
label
,
pConn
,
(
void
*
)
pHead
->
ahandle
,
TMSG_INFO
(
pHead
->
msgType
),
ipstr
,
terrno
,
pRecv
->
msgLen
,
pHead
->
sourceId
,
pHead
->
destId
,
pHead
->
tranId
,
pHead
->
code
);
}
else
{
tDebug
(
"%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x"
,
pRpc
->
label
,
pConn
,
(
void
*
)
pHead
->
ahandle
,
pHead
->
msgType
,
ipstr
,
terrno
,
pRecv
->
msgLen
,
pHead
->
sourceId
,
pHead
->
destId
,
pHead
->
tranId
,
pHead
->
code
);
tDebug
(
"%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x"
,
pRpc
->
label
,
pConn
,
(
void
*
)
pHead
->
ahandle
,
pHead
->
msgType
,
ipstr
,
terrno
,
pRecv
->
msgLen
,
pHead
->
sourceId
,
pHead
->
destId
,
pHead
->
tranId
,
pHead
->
code
);
}
int32_t
code
=
terrno
;
...
...
@@ -1118,9 +1250,11 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
rpcCloseConn
(
pConn
);
}
if
(
TMSG_INDEX
(
pHead
->
msgType
)
+
1
>
1
&&
TMSG_INDEX
(
pHead
->
msgType
)
+
1
<
TDMT_MAX
)
{
tDebug
(
"%s %p %p, %s is sent with error code:0x%x"
,
pRpc
->
label
,
pConn
,
(
void
*
)
pHead
->
ahandle
,
TMSG_INFO
(
pHead
->
msgType
+
1
),
code
);
tDebug
(
"%s %p %p, %s is sent with error code:0x%x"
,
pRpc
->
label
,
pConn
,
(
void
*
)
pHead
->
ahandle
,
TMSG_INFO
(
pHead
->
msgType
+
1
),
code
);
}
else
{
tError
(
"%s %p %p, %s is sent with error code:0x%x"
,
pRpc
->
label
,
pConn
,
(
void
*
)
pHead
->
ahandle
,
TMSG_INFO
(
pHead
->
msgType
),
code
);
tError
(
"%s %p %p, %s is sent with error code:0x%x"
,
pRpc
->
label
,
pConn
,
(
void
*
)
pHead
->
ahandle
,
TMSG_INFO
(
pHead
->
msgType
),
code
);
}
}
}
else
{
// msg is passed to app only parsing is ok
...
...
@@ -1144,8 +1278,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
}
else
{
// for asynchronous API
SEpSet
*
pEpSet
=
NULL
;
if
(
pContext
->
epSet
.
inUse
!=
pContext
->
oldInUse
||
pContext
->
redirect
)
pEpSet
=
&
pContext
->
epSet
;
if
(
pContext
->
epSet
.
inUse
!=
pContext
->
oldInUse
||
pContext
->
redirect
)
pEpSet
=
&
pContext
->
epSet
;
(
*
pRpc
->
cfp
)(
pRpc
->
parent
,
pMsg
,
pEpSet
);
}
...
...
@@ -1155,7 +1288,6 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
}
static
void
rpcProcessIncomingMsg
(
SRpcConn
*
pConn
,
SRpcHead
*
pHead
,
SRpcReqContext
*
pContext
)
{
SRpcInfo
*
pRpc
=
pConn
->
pRpc
;
SRpcMsg
rpcMsg
;
...
...
@@ -1180,14 +1312,15 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
// for UDP, port may be changed by server, the port in epSet shall be used for cache
if
(
pHead
->
code
!=
TSDB_CODE_RPC_TOO_SLOW
)
{
rpcAddConnIntoCache
(
pRpc
->
pCache
,
pConn
,
pConn
->
peerFqdn
,
pContext
->
epSet
.
port
[
pContext
->
epSet
.
inUse
],
pConn
->
connType
);
rpcAddConnIntoCache
(
pRpc
->
pCache
,
pConn
,
pConn
->
peerFqdn
,
pContext
->
epSet
.
port
[
pContext
->
epSet
.
inUse
],
pConn
->
connType
);
}
else
{
rpcCloseConn
(
pConn
);
}
if
(
pHead
->
code
==
TSDB_CODE_RPC_REDIRECT
)
{
pContext
->
numOfTry
=
0
;
SEpSet
*
pEpSet
=
(
SEpSet
*
)
pHead
->
content
;
SEpSet
*
pEpSet
=
(
SEpSet
*
)
pHead
->
content
;
if
(
pEpSet
->
numOfEps
>
0
)
{
memcpy
(
&
pContext
->
epSet
,
pHead
->
content
,
sizeof
(
pContext
->
epSet
));
tDebug
(
"%s, redirect is received, numOfEps:%d inUse:%d"
,
pConn
->
info
,
pContext
->
epSet
.
numOfEps
,
...
...
@@ -1200,7 +1333,8 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
}
rpcSendReqToServer
(
pRpc
,
pContext
);
rpcFreeCont
(
rpcMsg
.
pCont
);
}
else
if
(
pHead
->
code
==
TSDB_CODE_RPC_NOT_READY
||
pHead
->
code
==
TSDB_CODE_APP_NOT_READY
||
pHead
->
code
==
TSDB_CODE_DND_OFFLINE
)
{
}
else
if
(
pHead
->
code
==
TSDB_CODE_RPC_NOT_READY
||
pHead
->
code
==
TSDB_CODE_APP_NOT_READY
||
pHead
->
code
==
TSDB_CODE_DND_OFFLINE
)
{
pContext
->
code
=
pHead
->
code
;
rpcProcessConnError
(
pContext
,
NULL
);
rpcFreeCont
(
rpcMsg
.
pCont
);
...
...
@@ -1218,7 +1352,7 @@ static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) {
memset
(
msg
,
0
,
sizeof
(
SRpcHead
));
pHead
=
(
SRpcHead
*
)
msg
;
pHead
->
version
=
1
;
pHead
->
msgType
=
pConn
->
inType
+
1
;
pHead
->
msgType
=
pConn
->
inType
+
1
;
pHead
->
spi
=
pConn
->
spi
;
pHead
->
encrypt
=
0
;
pHead
->
tranId
=
pConn
->
inTranId
;
...
...
@@ -1258,7 +1392,7 @@ static void rpcSendReqHead(SRpcConn *pConn) {
static
void
rpcSendErrorMsgToPeer
(
SRecvInfo
*
pRecv
,
int32_t
code
)
{
SRpcHead
*
pRecvHead
,
*
pReplyHead
;
char
msg
[
sizeof
(
SRpcHead
)
+
sizeof
(
SRpcDigest
)
+
sizeof
(
uint32_t
)
];
char
msg
[
sizeof
(
SRpcHead
)
+
sizeof
(
SRpcDigest
)
+
sizeof
(
uint32_t
)
];
uint32_t
timeStamp
;
int
msgLen
;
...
...
@@ -1295,7 +1429,7 @@ static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) {
static
void
rpcSendReqToServer
(
SRpcInfo
*
pRpc
,
SRpcReqContext
*
pContext
)
{
SRpcHead
*
pHead
=
rpcHeadFromCont
(
pContext
->
pCont
);
char
*
msg
=
(
char
*
)
pHead
;
char
*
msg
=
(
char
*
)
pHead
;
int
msgLen
=
rpcMsgLenFromCont
(
pContext
->
contLen
);
tmsg_t
msgType
=
pContext
->
msgType
;
...
...
@@ -1317,7 +1451,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
pHead
->
msgType
=
msgType
;
pHead
->
encrypt
=
0
;
pConn
->
tranId
++
;
if
(
pConn
->
tranId
==
0
)
pConn
->
tranId
++
;
if
(
pConn
->
tranId
==
0
)
pConn
->
tranId
++
;
pHead
->
tranId
=
pConn
->
tranId
;
pHead
->
sourceId
=
pConn
->
ownId
;
pHead
->
destId
=
pConn
->
peerId
;
...
...
@@ -1346,18 +1480,16 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
msgLen
=
rpcAddAuthPart
(
pConn
,
msg
,
msgLen
);
if
(
rpcIsReq
(
pHead
->
msgType
))
{
tDebug
(
"%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d"
,
pConn
->
info
,
TMSG_INFO
(
pHead
->
msgType
),
pConn
->
peerFqdn
,
pConn
->
peerPort
,
msgLen
,
pHead
->
sourceId
,
pHead
->
destId
,
pHead
->
tranId
);
if
(
rpcIsReq
(
pHead
->
msgType
))
{
tDebug
(
"%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d"
,
pConn
->
info
,
TMSG_INFO
(
pHead
->
msgType
),
pConn
->
peerFqdn
,
pConn
->
peerPort
,
msgLen
,
pHead
->
sourceId
,
pHead
->
destId
,
pHead
->
tranId
);
}
else
{
if
(
pHead
->
code
==
0
)
pConn
->
secured
=
1
;
// for success response, set link as secured
tDebug
(
"%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d"
,
pConn
->
info
,
TMSG_INFO
(
pHead
->
msgType
),
pConn
->
peerIp
,
pConn
->
peerPort
,
htonl
(
pHead
->
code
),
msgLen
,
pHead
->
sourceId
,
pHead
->
destId
,
pHead
->
tranId
);
tDebug
(
"%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d"
,
pConn
->
info
,
TMSG_INFO
(
pHead
->
msgType
),
pConn
->
peerIp
,
pConn
->
peerPort
,
htonl
(
pHead
->
code
),
msgLen
,
pHead
->
sourceId
,
pHead
->
destId
,
pHead
->
tranId
);
}
//tTrace("connection type is: %d", pConn->connType);
//
tTrace("connection type is: %d", pConn->connType);
writtenLen
=
(
*
taosSendData
[
pConn
->
connType
])(
pConn
->
peerIp
,
pConn
->
peerPort
,
pHead
,
msgLen
,
pConn
->
chandle
);
if
(
writtenLen
!=
msgLen
)
{
...
...
@@ -1369,7 +1501,7 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
static
void
rpcProcessConnError
(
void
*
param
,
void
*
id
)
{
SRpcReqContext
*
pContext
=
(
SRpcReqContext
*
)
param
;
SRpcInfo
*
pRpc
=
pContext
->
pRpc
;
SRpcInfo
*
pRpc
=
pContext
->
pRpc
;
SRpcMsg
rpcMsg
;
if
(
pRpc
==
NULL
)
{
...
...
@@ -1379,7 +1511,7 @@ static void rpcProcessConnError(void *param, void *id) {
tDebug
(
"%s %p, connection error happens"
,
pRpc
->
label
,
pContext
->
ahandle
);
if
(
pContext
->
numOfTry
>=
pContext
->
epSet
.
numOfEps
||
pContext
->
msgType
==
TDMT_VND_FETCH
)
{
rpcMsg
.
msgType
=
pContext
->
msgType
+
1
;
rpcMsg
.
msgType
=
pContext
->
msgType
+
1
;
rpcMsg
.
ahandle
=
pContext
->
ahandle
;
rpcMsg
.
code
=
pContext
->
code
;
rpcMsg
.
pCont
=
NULL
;
...
...
@@ -1411,7 +1543,8 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) {
pConn
->
pTimer
=
taosTmrStart
(
rpcProcessRetryTimer
,
tsRpcTimer
,
pConn
,
pRpc
->
tmrCtrl
);
}
else
{
// close the connection
tDebug
(
"%s, failed to send msg:%s to %s:%hu"
,
pConn
->
info
,
TMSG_INFO
(
pConn
->
outType
),
pConn
->
peerFqdn
,
pConn
->
peerPort
);
tDebug
(
"%s, failed to send msg:%s to %s:%hu"
,
pConn
->
info
,
TMSG_INFO
(
pConn
->
outType
),
pConn
->
peerFqdn
,
pConn
->
peerPort
);
if
(
pConn
->
pContext
)
{
pConn
->
pContext
->
code
=
TSDB_CODE_RPC_NETWORK_UNAVAIL
;
pConn
->
pContext
->
pConn
=
NULL
;
...
...
@@ -1460,7 +1593,7 @@ static void rpcProcessProgressTimer(void *param, void *tmrId) {
rpcUnlockConn
(
pConn
);
}
static
int32_t
rpcCompressRpcMsg
(
char
*
pCont
,
int32_t
contLen
)
{
static
int32_t
rpcCompressRpcMsg
(
char
*
pCont
,
int32_t
contLen
)
{
SRpcHead
*
pHead
=
rpcHeadFromCont
(
pCont
);
int32_t
finalLen
=
0
;
int
overhead
=
sizeof
(
SRpcComp
);
...
...
@@ -1469,7 +1602,7 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) {
return
contLen
;
}
char
*
buf
=
malloc
(
contLen
+
overhead
+
8
);
// 8 extra bytes
char
*
buf
=
malloc
(
contLen
+
overhead
+
8
);
// 8 extra bytes
if
(
buf
==
NULL
)
{
tError
(
"failed to allocate memory for rpc msg compression, contLen:%d"
,
contLen
);
return
contLen
;
...
...
@@ -1502,7 +1635,7 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) {
static
SRpcHead
*
rpcDecompressRpcMsg
(
SRpcHead
*
pHead
)
{
int
overhead
=
sizeof
(
SRpcComp
);
SRpcHead
*
pNewHead
=
NULL
;
uint8_t
*
pCont
=
pHead
->
content
;
uint8_t
*
pCont
=
pHead
->
content
;
SRpcComp
*
pComp
=
(
SRpcComp
*
)
pHead
->
content
;
if
(
pHead
->
comp
)
{
...
...
@@ -1516,7 +1649,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) {
if
(
pNewHead
)
{
int
compLen
=
rpcContLenFromMsg
(
pHead
->
msgLen
)
-
overhead
;
int
origLen
=
LZ4_decompress_safe
((
char
*
)(
pCont
+
overhead
),
(
char
*
)
pNewHead
->
content
,
compLen
,
contLen
);
int
origLen
=
LZ4_decompress_safe
((
char
*
)(
pCont
+
overhead
),
(
char
*
)
pNewHead
->
content
,
compLen
,
contLen
);
assert
(
origLen
==
contLen
);
memcpy
(
pNewHead
,
pHead
,
sizeof
(
SRpcHead
));
...
...
@@ -1582,19 +1715,19 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) {
SRpcHead
*
pHead
=
(
SRpcHead
*
)
msg
;
int
code
=
0
;
if
((
pConn
->
secured
&&
pHead
->
spi
==
0
)
||
(
pHead
->
spi
==
0
&&
pConn
->
spi
==
0
)){
if
((
pConn
->
secured
&&
pHead
->
spi
==
0
)
||
(
pHead
->
spi
==
0
&&
pConn
->
spi
==
0
))
{
// secured link, or no authentication
pHead
->
msgLen
=
(
int32_t
)
htonl
((
uint32_t
)
pHead
->
msgLen
);
// tTrace("%s, secured link, no auth is required", pConn->info);
return
0
;
}
if
(
!
rpcIsReq
(
pHead
->
msgType
)
)
{
if
(
!
rpcIsReq
(
pHead
->
msgType
)
)
{
// for response, if code is auth failure, it shall bypass the auth process
code
=
htonl
(
pHead
->
code
);
if
(
code
==
TSDB_CODE_RPC_INVALID_TIME_STAMP
||
code
==
TSDB_CODE_RPC_AUTH_FAILURE
||
code
==
TSDB_CODE_RPC_INVALID_VERSION
||
code
==
TSDB_CODE_
RPC_AUTH_REQUIRED
||
code
==
TSDB_CODE_
MND_USER_NOT_EXIST
||
code
==
TSDB_CODE_RPC_NOT_READY
)
{
code
==
TSDB_CODE_RPC_INVALID_VERSION
||
code
==
TSDB_CODE_RPC_AUTH_REQUIRED
||
code
==
TSDB_CODE_MND_USER_NOT_EXIST
||
code
==
TSDB_CODE_RPC_NOT_READY
)
{
pHead
->
msgLen
=
(
int32_t
)
htonl
((
uint32_t
)
pHead
->
msgLen
);
// tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code);
return
0
;
...
...
@@ -1613,12 +1746,12 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) {
tWarn
(
"%s, time diff:%d is too big, msg discarded"
,
pConn
->
info
,
delta
);
code
=
TSDB_CODE_RPC_INVALID_TIME_STAMP
;
}
else
{
if
(
rpcAuthenticateMsg
(
pHead
,
msgLen
-
TSDB_AUTH_LEN
,
pDigest
->
auth
,
pConn
->
secret
)
<
0
)
{
if
(
rpcAuthenticateMsg
(
pHead
,
msgLen
-
TSDB_AUTH_LEN
,
pDigest
->
auth
,
pConn
->
secret
)
<
0
)
{
tDebug
(
"%s, authentication failed, msg discarded"
,
pConn
->
info
);
code
=
TSDB_CODE_RPC_AUTH_FAILURE
;
}
else
{
pHead
->
msgLen
=
(
int32_t
)
htonl
((
uint32_t
)
pHead
->
msgLen
)
-
sizeof
(
SRpcDigest
);
if
(
!
rpcIsReq
(
pHead
->
msgType
)
)
pConn
->
secured
=
1
;
// link is secured for client
if
(
!
rpcIsReq
(
pHead
->
msgType
)
)
pConn
->
secured
=
1
;
// link is secured for client
// tTrace("%s, message is authenticated", pConn->info);
}
}
...
...
@@ -1647,13 +1780,9 @@ static void rpcUnlockConn(SRpcConn *pConn) {
}
}
static
void
rpcAddRef
(
SRpcInfo
*
pRpc
)
{
atomic_add_fetch_32
(
&
pRpc
->
refCount
,
1
);
}
static
void
rpcAddRef
(
SRpcInfo
*
pRpc
)
{
atomic_add_fetch_32
(
&
pRpc
->
refCount
,
1
);
}
static
void
rpcDecRef
(
SRpcInfo
*
pRpc
)
{
static
void
rpcDecRef
(
SRpcInfo
*
pRpc
)
{
if
(
atomic_sub_fetch_32
(
&
pRpc
->
refCount
,
1
)
==
0
)
{
rpcCloseConnCache
(
pRpc
->
pCache
);
taosHashCleanup
(
pRpc
->
hash
);
...
...
@@ -1668,4 +1797,4 @@ static void rpcDecRef(SRpcInfo *pRpc)
atomic_sub_fetch_32
(
&
tsRpcNum
,
1
);
}
}
#endif
source/libs/transport/src/rpcTcp.c
浏览文件 @
b9450007
...
...
@@ -13,24 +13,28 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "rpcTcp.h"
#include <uv.h>
#include "os.h"
#include "tutil.h"
#include "rpcHead.h"
#include "rpcLog.h"
#include "taosdef.h"
#include "taoserror.h"
#include "
rpcLog
.h"
#include "rpcHead.h"
#i
nclude "rpcTcp.h"
#include "
tutil
.h"
#i
fdef USE_UV
#else
typedef
struct
SFdObj
{
void
*
signature
;
void
*
signature
;
SOCKET
fd
;
// TCP socket FD
void
*
thandle
;
// handle from upper layer, like TAOS
void
*
thandle
;
// handle from upper layer, like TAOS
uint32_t
ip
;
uint16_t
port
;
int16_t
closedByApp
;
// 1: already closed by App
struct
SThreadObj
*
pThreadObj
;
struct
SFdObj
*
prev
;
struct
SFdObj
*
next
;
struct
SFdObj
*
prev
;
struct
SFdObj
*
next
;
}
SFdObj
;
typedef
struct
SThreadObj
{
...
...
@@ -43,7 +47,7 @@ typedef struct SThreadObj {
int
numOfFds
;
int
threadId
;
char
label
[
TSDB_LABEL_LEN
];
void
*
shandle
;
// handle passed by upper layer during server initialization
void
*
shandle
;
// handle passed by upper layer during server initialization
void
*
(
*
processData
)(
SRecvInfo
*
pPacket
);
}
SThreadObj
;
...
...
@@ -67,11 +71,11 @@ typedef struct {
pthread_t
thread
;
}
SServerObj
;
static
void
*
taosProcessTcpData
(
void
*
param
);
static
void
*
taosProcessTcpData
(
void
*
param
);
static
SFdObj
*
taosMallocFdObj
(
SThreadObj
*
pThreadObj
,
SOCKET
fd
);
static
void
taosFreeFdObj
(
SFdObj
*
pFdObj
);
static
void
taosReportBrokenLink
(
SFdObj
*
pFdObj
);
static
void
*
taosAcceptTcpConnection
(
void
*
arg
);
static
void
*
taosAcceptTcpConnection
(
void
*
arg
);
void
*
taosInitTcpServer
(
uint32_t
ip
,
uint16_t
port
,
char
*
label
,
int
numOfThreads
,
void
*
fp
,
void
*
shandle
)
{
SServerObj
*
pServerObj
;
...
...
@@ -110,7 +114,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
if
(
pThreadObj
==
NULL
)
{
tError
(
"TCP:%s no enough memory"
,
label
);
terrno
=
TAOS_SYSTEM_ERROR
(
errno
);
for
(
int
j
=
0
;
j
<
i
;
++
j
)
free
(
pServerObj
->
pThreadObj
[
j
]);
for
(
int
j
=
0
;
j
<
i
;
++
j
)
free
(
pServerObj
->
pThreadObj
[
j
]);
free
(
pServerObj
->
pThreadObj
);
free
(
pServerObj
);
return
NULL
;
...
...
@@ -172,8 +176,10 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
return
(
void
*
)
pServerObj
;
}
static
void
taosStopTcpThread
(
SThreadObj
*
pThreadObj
)
{
if
(
pThreadObj
==
NULL
)
{
return
;}
static
void
taosStopTcpThread
(
SThreadObj
*
pThreadObj
)
{
if
(
pThreadObj
==
NULL
)
{
return
;
}
// save thread into local variable and signal thread to stop
pthread_t
thread
=
pThreadObj
->
thread
;
if
(
!
taosCheckPthreadValid
(
thread
))
{
...
...
@@ -227,8 +233,8 @@ static void *taosAcceptTcpConnection(void *arg) {
SOCKET
connFd
=
-
1
;
struct
sockaddr_in
caddr
;
int
threadId
=
0
;
SThreadObj
*
pThreadObj
;
SServerObj
*
pServerObj
;
SThreadObj
*
pThreadObj
;
SServerObj
*
pServerObj
;
pServerObj
=
(
SServerObj
*
)
arg
;
tDebug
(
"%s TCP server is ready, ip:0x%x:%hu"
,
pServerObj
->
label
,
pServerObj
->
ip
,
pServerObj
->
port
);
...
...
@@ -253,7 +259,7 @@ static void *taosAcceptTcpConnection(void *arg) {
}
taosKeepTcpAlive
(
connFd
);
struct
timeval
to
=
{
5
,
0
};
struct
timeval
to
=
{
5
,
0
};
int32_t
ret
=
taosSetSockOpt
(
connFd
,
SOL_SOCKET
,
SO_RCVTIMEO
,
&
to
,
sizeof
(
to
));
if
(
ret
!=
0
)
{
taosCloseSocket
(
connFd
);
...
...
@@ -262,7 +268,6 @@ static void *taosAcceptTcpConnection(void *arg) {
continue
;
}
// pick up the thread to handle this connection
pThreadObj
=
pServerObj
->
pThreadObj
[
threadId
];
...
...
@@ -297,7 +302,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
tstrncpy
(
pClientObj
->
label
,
label
,
sizeof
(
pClientObj
->
label
));
pClientObj
->
numOfThreads
=
numOfThreads
;
pClientObj
->
pThreadObj
=
(
SThreadObj
**
)
calloc
(
numOfThreads
,
sizeof
(
SThreadObj
*
));
pClientObj
->
pThreadObj
=
(
SThreadObj
**
)
calloc
(
numOfThreads
,
sizeof
(
SThreadObj
*
));
if
(
pClientObj
->
pThreadObj
==
NULL
)
{
tError
(
"TCP:%s no enough memory"
,
label
);
tfree
(
pClientObj
);
...
...
@@ -314,7 +319,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
if
(
pThreadObj
==
NULL
)
{
tError
(
"TCP:%s no enough memory"
,
label
);
terrno
=
TAOS_SYSTEM_ERROR
(
errno
);
for
(
int
j
=
0
;
j
<
i
;
++
j
)
free
(
pClientObj
->
pThreadObj
[
j
]);
for
(
int
j
=
0
;
j
<
i
;
++
j
)
free
(
pClientObj
->
pThreadObj
[
j
]);
free
(
pClientObj
);
pthread_attr_destroy
(
&
thattr
);
return
NULL
;
...
...
@@ -364,14 +369,14 @@ void taosStopTcpClient(void *chandle) {
if
(
pClientObj
==
NULL
)
return
;
tDebug
(
"%s TCP client is stopped"
,
pClientObj
->
label
);
tDebug
(
"%s TCP client is stopped"
,
pClientObj
->
label
);
}
void
taosCleanUpTcpClient
(
void
*
chandle
)
{
SClientObj
*
pClientObj
=
chandle
;
if
(
pClientObj
==
NULL
)
return
;
for
(
int
i
=
0
;
i
<
pClientObj
->
numOfThreads
;
++
i
)
{
SThreadObj
*
pThreadObj
=
pClientObj
->
pThreadObj
[
i
];
SThreadObj
*
pThreadObj
=
pClientObj
->
pThreadObj
[
i
];
taosStopTcpThread
(
pThreadObj
);
}
...
...
@@ -381,7 +386,7 @@ void taosCleanUpTcpClient(void *chandle) {
}
void
*
taosOpenTcpClientConnection
(
void
*
shandle
,
void
*
thandle
,
uint32_t
ip
,
uint16_t
port
)
{
SClientObj
*
pClientObj
=
shandle
;
SClientObj
*
pClientObj
=
shandle
;
int32_t
index
=
atomic_load_32
(
&
pClientObj
->
index
)
%
pClientObj
->
numOfThreads
;
atomic_store_32
(
&
pClientObj
->
index
,
index
+
1
);
SThreadObj
*
pThreadObj
=
pClientObj
->
pThreadObj
[
index
];
...
...
@@ -396,8 +401,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
struct
sockaddr_in
sin
;
uint16_t
localPort
=
0
;
unsigned
int
addrlen
=
sizeof
(
sin
);
if
(
getsockname
(
fd
,
(
struct
sockaddr
*
)
&
sin
,
&
addrlen
)
==
0
&&
sin
.
sin_family
==
AF_INET
&&
addrlen
==
sizeof
(
sin
))
{
if
(
getsockname
(
fd
,
(
struct
sockaddr
*
)
&
sin
,
&
addrlen
)
==
0
&&
sin
.
sin_family
==
AF_INET
&&
addrlen
==
sizeof
(
sin
))
{
localPort
=
(
uint16_t
)
ntohs
(
sin
.
sin_port
);
}
...
...
@@ -407,8 +411,8 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
pFdObj
->
thandle
=
thandle
;
pFdObj
->
port
=
port
;
pFdObj
->
ip
=
ip
;
tDebug
(
"%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d"
,
pThreadObj
->
label
,
thandle
,
ip
,
port
,
localPort
,
pFdObj
,
pThreadObj
->
numOfFds
);
tDebug
(
"%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d"
,
pThreadObj
->
label
,
thandle
,
ip
,
port
,
localPort
,
pFdObj
,
pThreadObj
->
numOfFds
);
}
else
{
tError
(
"%s failed to malloc client FdObj(%s)"
,
pThreadObj
->
label
,
strerror
(
errno
));
taosCloseSocket
(
fd
);
...
...
@@ -441,7 +445,6 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand
}
static
void
taosReportBrokenLink
(
SFdObj
*
pFdObj
)
{
SThreadObj
*
pThreadObj
=
pFdObj
->
pThreadObj
;
// notify the upper layer, so it will clean the associated context
...
...
@@ -466,7 +469,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) {
static
int
taosReadTcpData
(
SFdObj
*
pFdObj
,
SRecvInfo
*
pInfo
)
{
SRpcHead
rpcHead
;
int32_t
msgLen
,
leftLen
,
retLen
,
headLen
;
char
*
buffer
,
*
msg
;
char
*
buffer
,
*
msg
;
SThreadObj
*
pThreadObj
=
pFdObj
->
pThreadObj
;
...
...
@@ -483,7 +486,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
tError
(
"%s %p TCP malloc(size:%d) fail"
,
pThreadObj
->
label
,
pFdObj
->
thandle
,
msgLen
);
return
-
1
;
}
else
{
tTrace
(
"%s %p read data, FD:%p fd:%d TCP malloc mem:%p"
,
pThreadObj
->
label
,
pFdObj
->
thandle
,
pFdObj
,
pFdObj
->
fd
,
buffer
);
tTrace
(
"%s %p read data, FD:%p fd:%d TCP malloc mem:%p"
,
pThreadObj
->
label
,
pFdObj
->
thandle
,
pFdObj
,
pFdObj
->
fd
,
buffer
);
}
msg
=
buffer
+
tsRpcOverhead
;
...
...
@@ -491,8 +495,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
retLen
=
taosReadMsg
(
pFdObj
->
fd
,
msg
+
headLen
,
leftLen
);
if
(
leftLen
!=
retLen
)
{
tError
(
"%s %p read error, leftLen:%d retLen:%d FD:%p"
,
pThreadObj
->
label
,
pFdObj
->
thandle
,
leftLen
,
retLen
,
pFdObj
);
tError
(
"%s %p read error, leftLen:%d retLen:%d FD:%p"
,
pThreadObj
->
label
,
pFdObj
->
thandle
,
leftLen
,
retLen
,
pFdObj
);
free
(
buffer
);
return
-
1
;
}
...
...
@@ -519,8 +522,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
#define maxEvents 10
static
void
*
taosProcessTcpData
(
void
*
param
)
{
SThreadObj
*
pThreadObj
=
param
;
SFdObj
*
pFdObj
;
SThreadObj
*
pThreadObj
=
param
;
SFdObj
*
pFdObj
;
struct
epoll_event
events
[
maxEvents
];
SRecvInfo
recvInfo
;
...
...
@@ -569,7 +572,7 @@ static void *taosProcessTcpData(void *param) {
if
(
pThreadObj
->
stop
)
break
;
}
if
(
pThreadObj
->
pollFd
>=
0
)
{
if
(
pThreadObj
->
pollFd
>=
0
)
{
EpollClose
(
pThreadObj
->
pollFd
);
pThreadObj
->
pollFd
=
-
1
;
}
...
...
@@ -620,7 +623,6 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) {
}
static
void
taosFreeFdObj
(
SFdObj
*
pFdObj
)
{
if
(
pFdObj
==
NULL
)
return
;
if
(
pFdObj
->
signature
!=
pFdObj
)
return
;
...
...
@@ -638,8 +640,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pThreadObj
->
numOfFds
--
;
if
(
pThreadObj
->
numOfFds
<
0
)
tError
(
"%s %p TCP thread:%d, number of FDs is negative!!!"
,
pThreadObj
->
label
,
pFdObj
->
thandle
,
pThreadObj
->
threadId
);
tError
(
"%s %p TCP thread:%d, number of FDs is negative!!!"
,
pThreadObj
->
label
,
pFdObj
->
thandle
,
pThreadObj
->
threadId
);
if
(
pFdObj
->
prev
)
{
(
pFdObj
->
prev
)
->
next
=
pFdObj
->
next
;
...
...
@@ -653,8 +655,10 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pthread_mutex_unlock
(
&
pThreadObj
->
mutex
);
tDebug
(
"%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d"
,
pThreadObj
->
label
,
pFdObj
->
thandle
,
pFdObj
,
pFdObj
->
fd
,
pThreadObj
->
numOfFds
);
tDebug
(
"%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d"
,
pThreadObj
->
label
,
pFdObj
->
thandle
,
pFdObj
,
pFdObj
->
fd
,
pThreadObj
->
numOfFds
);
tfree
(
pFdObj
);
}
#endif
source/libs/transport/src/rpcUdp.c
浏览文件 @
b9450007
...
...
@@ -13,15 +13,18 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "rpcUdp.h"
#include "os.h"
#include "
ttimer
.h"
#include "
tutil
.h"
#include "
rpcHead
.h"
#include "
rpcLog
.h"
#include "taosdef.h"
#include "taoserror.h"
#include "rpcLog.h"
#include "rpcUdp.h"
#include "rpcHead.h"
#include "ttimer.h"
#include "tutil.h"
#ifdef USE_UV
// no support upd currently
#else
#define RPC_MAX_UDP_CONNS 256
#define RPC_MAX_UDP_PKTS 1000
#define RPC_UDP_BUF_TIME 5 // mseconds
...
...
@@ -34,9 +37,9 @@ typedef struct {
uint16_t
localPort
;
// local port
char
label
[
TSDB_LABEL_LEN
];
// copy from udpConnSet;
pthread_t
thread
;
void
*
hash
;
void
*
shandle
;
// handle passed by upper layer during server initialization
void
*
pSet
;
void
*
hash
;
void
*
shandle
;
// handle passed by upper layer during server initialization
void
*
pSet
;
void
*
(
*
processData
)(
SRecvInfo
*
pRecv
);
char
*
buffer
;
// buffer to receive data
}
SUdpConn
;
...
...
@@ -46,7 +49,7 @@ typedef struct {
int
server
;
uint32_t
ip
;
// local IP
uint16_t
port
;
// local Port
void
*
shandle
;
// handle passed by upper layer during server initialization
void
*
shandle
;
// handle passed by upper layer during server initialization
int
threads
;
char
label
[
TSDB_LABEL_LEN
];
void
*
(
*
fp
)(
SRecvInfo
*
pPacket
);
...
...
@@ -56,7 +59,7 @@ typedef struct {
static
void
*
taosRecvUdpData
(
void
*
param
);
void
*
taosInitUdpConnection
(
uint32_t
ip
,
uint16_t
port
,
char
*
label
,
int
threads
,
void
*
fp
,
void
*
shandle
)
{
SUdpConn
*
pConn
;
SUdpConn
*
pConn
;
SUdpConnSet
*
pSet
;
int
size
=
(
int
)
sizeof
(
SUdpConnSet
)
+
threads
*
(
int
)
sizeof
(
SUdpConn
);
...
...
@@ -98,8 +101,8 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
struct
sockaddr_in
sin
;
unsigned
int
addrlen
=
sizeof
(
sin
);
if
(
getsockname
(
pConn
->
fd
,
(
struct
sockaddr
*
)
&
sin
,
&
addrlen
)
==
0
&&
sin
.
sin_family
==
AF_INET
&&
addrlen
==
sizeof
(
sin
))
{
if
(
getsockname
(
pConn
->
fd
,
(
struct
sockaddr
*
)
&
sin
,
&
addrlen
)
==
0
&&
sin
.
sin_family
==
AF_INET
&&
addrlen
==
sizeof
(
sin
))
{
pConn
->
localPort
=
(
uint16_t
)
ntohs
(
sin
.
sin_port
);
}
...
...
@@ -130,14 +133,14 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
void
taosStopUdpConnection
(
void
*
handle
)
{
SUdpConnSet
*
pSet
=
(
SUdpConnSet
*
)
handle
;
SUdpConn
*
pConn
;
SUdpConn
*
pConn
;
if
(
pSet
==
NULL
)
return
;
for
(
int
i
=
0
;
i
<
pSet
->
threads
;
++
i
)
{
pConn
=
pSet
->
udpConn
+
i
;
if
(
pConn
->
fd
>=
0
)
shutdown
(
pConn
->
fd
,
SHUT_RDWR
);
if
(
pConn
->
fd
>=
0
)
taosCloseSocket
(
pConn
->
fd
);
if
(
pConn
->
fd
>=
0
)
shutdown
(
pConn
->
fd
,
SHUT_RDWR
);
if
(
pConn
->
fd
>=
0
)
taosCloseSocket
(
pConn
->
fd
);
pConn
->
fd
=
-
1
;
}
...
...
@@ -155,13 +158,13 @@ void taosStopUdpConnection(void *handle) {
void
taosCleanUpUdpConnection
(
void
*
handle
)
{
SUdpConnSet
*
pSet
=
(
SUdpConnSet
*
)
handle
;
SUdpConn
*
pConn
;
SUdpConn
*
pConn
;
if
(
pSet
==
NULL
)
return
;
for
(
int
i
=
0
;
i
<
pSet
->
threads
;
++
i
)
{
pConn
=
pSet
->
udpConn
+
i
;
if
(
pConn
->
fd
>=
0
)
taosCloseSocket
(
pConn
->
fd
);
if
(
pConn
->
fd
>=
0
)
taosCloseSocket
(
pConn
->
fd
);
}
tDebug
(
"%s UDP is cleaned up"
,
pSet
->
label
);
...
...
@@ -182,7 +185,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
}
static
void
*
taosRecvUdpData
(
void
*
param
)
{
SUdpConn
*
pConn
=
param
;
SUdpConn
*
pConn
=
param
;
struct
sockaddr_in
sourceAdd
;
ssize_t
dataLen
;
unsigned
int
addLen
;
...
...
@@ -218,7 +221,7 @@ static void *taosRecvUdpData(void *param) {
}
int32_t
size
=
dataLen
+
tsRpcOverhead
;
char
*
tmsg
=
malloc
(
size
);
char
*
tmsg
=
malloc
(
size
);
if
(
NULL
==
tmsg
)
{
tError
(
"%s failed to allocate memory, size:%"
PRId64
,
pConn
->
label
,
(
int64_t
)
dataLen
);
continue
;
...
...
@@ -257,4 +260,4 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c
return
ret
;
}
#endif
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录