提交 ba24ba19 编写于 作者: L liuyq-617

Merge branch 'develop' into test/jenkins

......@@ -48,7 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat
cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin
#cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver
......
......@@ -58,7 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri
cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
#cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver
cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include
......@@ -135,7 +135,7 @@ if [ $1 -eq 0 ];then
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
#${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${cfg_link_dir}/* || :
${csudo} rm -f ${inc_link_dir}/taos.h || :
${csudo} rm -f ${inc_link_dir}/taoserror.h || :
......
......@@ -45,7 +45,8 @@ if [ "$osType" != "Darwin" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
else
bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
#bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"
else
......
......@@ -36,7 +36,8 @@ if [ "$pagMode" == "lite" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
else
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
#bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"
......
......@@ -330,6 +330,7 @@ typedef struct STscObj {
char writeAuth : 1;
char superAuth : 1;
uint32_t connId;
uint64_t rid; // ref ID returned by taosAddRef
struct SSqlObj * pHb;
struct SSqlObj * sqlList;
struct SSqlStream *streamList;
......@@ -348,7 +349,7 @@ typedef struct SSqlObj {
void *signature;
pthread_t owner; // owner of sql object, by which it is executed
STscObj *pTscObj;
void *pRpcCtx;
int64_t rpcRid;
void (*fp)();
void (*fetchFp)();
void *param;
......
......@@ -16,6 +16,7 @@
#define _BSD_SOURCE
#define _XOPEN_SOURCE 500
#define _DEFAULT_SOURCE
#define _GNU_SOURCE
#include "os.h"
#include "qAst.h"
......@@ -2037,7 +2038,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
SColumnIndex index = {.tableIndex = j, .columnIndex = i};
char name[TSDB_COL_NAME_LEN] = {0};
SStrToken t = {.z = pSchema->name, .n = (uint32_t)strnlen(pSchema->name, TSDB_COL_NAME_LEN)};
SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)};
setResultColName(name, pItem, cvtFunc.originFuncId, &t);
if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) {
......
......@@ -182,27 +182,23 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
int32_t waitingDuring = tsShellActivityTimer * 500;
tscDebug("%p send heartbeat in %dms", pSql, waitingDuring);
taosTmrReset(tscProcessActivityTimer, waitingDuring, pObj, tscTmr, &pObj->pTimer);
taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer);
} else {
tscDebug("%p start to close tscObj:%p, not send heartbeat again", pSql, pObj);
}
}
void tscProcessActivityTimer(void *handle, void *tmrId) {
STscObj *pObj = (STscObj *)handle;
int ret = taosAcquireRef(tscRefId, pObj);
if (ret < 0) {
tscTrace("%p failed to acquire TSC obj, reason:%s", pObj, tstrerror(ret));
return;
}
int64_t rid = (int64_t) handle;
STscObj *pObj = taosAcquireRef(tscRefId, rid);
if (pObj == NULL) return;
SSqlObj* pHB = pObj->pHb;
void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
if (p == NULL) {
tscWarn("%p HB object has been released already", pHB);
taosReleaseRef(tscRefId, pObj);
taosReleaseRef(tscRefId, pObj->rid);
return;
}
......@@ -216,7 +212,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) {
tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
}
taosReleaseRef(tscRefId, pObj);
taosReleaseRef(tscRefId, rid);
}
int tscSendMsgToServer(SSqlObj *pSql) {
......@@ -241,7 +237,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.pCont = pMsg,
.contLen = pSql->cmd.payloadLen,
.ahandle = pSql,
.handle = &pSql->pRpcCtx,
.handle = NULL,
.code = 0
};
......@@ -249,7 +245,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
// Otherwise, the pSql object may have been released already during the response function, which is
// processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
// cause crash.
rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
pSql->rpcRid = rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
return TSDB_CODE_SUCCESS;
}
......@@ -269,7 +265,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
SSqlCmd *pCmd = &pSql->cmd;
assert(*pSql->self == pSql);
pSql->pRpcCtx = NULL;
pSql->rpcRid = -1;
if (pObj->signature != pObj) {
tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
......@@ -1722,8 +1718,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name,
strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000);
// todo handle out of memory case
if (pTableMetaInfo->pTableMeta == NULL) {
free(pTableMeta);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
......@@ -2026,7 +2021,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
createHBObj(pObj);
//launch a timer to send heartbeat to maintain the connection and send status to mnode
taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer);
return 0;
}
......
......@@ -161,7 +161,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
registerSqlObj(pSql);
tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
taosAddRef(tscRefId, pObj);
pObj->rid = taosAddRef(tscRefId, pObj);
return pSql;
}
......@@ -279,9 +279,9 @@ void taos_close(TAOS *taos) {
SSqlObj* pHb = pObj->pHb;
if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
if (pHb->pRpcCtx != NULL) { // wait for rsp from dnode
rpcCancelRequest(pHb->pRpcCtx);
pHb->pRpcCtx = NULL;
if (pHb->rpcRid > 0) { // wait for rsp from dnode
rpcCancelRequest(pHb->rpcRid);
pHb->rpcRid = -1;
}
tscDebug("%p HB is freed", pHb);
......@@ -298,7 +298,7 @@ void taos_close(TAOS *taos) {
tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
taosRemoveRef(tscRefId, pObj);
taosRemoveRef(tscRefId, pObj->rid);
}
void waitForQueryRsp(void *param, TAOS_RES *tres, int code) {
......@@ -748,9 +748,9 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
assert(pSubObj->self == (SSqlObj**) p);
pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
if (pSubObj->pRpcCtx != NULL) {
rpcCancelRequest(pSubObj->pRpcCtx);
pSubObj->pRpcCtx = NULL;
if (pSubObj->rpcRid > 0) {
rpcCancelRequest(pSubObj->rpcRid);
pSubObj->rpcRid = -1;
}
tscQueueAsyncRes(pSubObj);
......@@ -775,7 +775,7 @@ void taos_stop_query(TAOS_RES *res) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
assert(pSql->pRpcCtx == NULL);
assert(pSql->rpcRid <= 0);
tscKillSTableQuery(pSql);
} else {
if (pSql->cmd.command < TSDB_SQL_LOCAL) {
......@@ -784,9 +784,9 @@ void taos_stop_query(TAOS_RES *res) {
* reset and freed in the processMsgFromServer function, and causes the invalid
* write problem for rpcCancelRequest.
*/
if (pSql->pRpcCtx != NULL) {
rpcCancelRequest(pSql->pRpcCtx);
pSql->pRpcCtx = NULL;
if (pSql->rpcRid > 0) {
rpcCancelRequest(pSql->rpcRid);
pSql->rpcRid = -1;
}
tscQueueAsyncRes(pSql);
......
......@@ -36,7 +36,7 @@ void * tscTmr;
void * tscQhandle;
void * tscCheckDiskUsageTmr;
int tsInsertHeadSize;
int tscRefId;
int tscRefId = -1;
int tscNumOfThreads;
......
......@@ -376,7 +376,7 @@ void tscFreeRegisteredSqlObj(void *pSql) {
tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref);
if (ref == 0) {
tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj);
taosRemoveRef(tscRefId, pTscObj);
taosRemoveRef(tscRefId, pTscObj->rid);
}
}
......
......@@ -39,16 +39,16 @@
#define cTrace(...) { if (cqDebugFlag & DEBUG_TRACE) { taosPrintLog("CQ ", cqDebugFlag, __VA_ARGS__); }}
typedef struct {
int vgId;
int32_t vgId;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_NAME_LEN];
FCqWrite cqWrite;
void *ahandle;
int num; // number of continuous streams
int32_t num; // number of continuous streams
struct SCqObj *pHead;
void *dbConn;
int master;
int32_t master;
void *tmrCtrl;
pthread_mutex_t mutex;
} SCqContext;
......@@ -57,7 +57,7 @@ typedef struct SCqObj {
tmr_h tmrId;
uint64_t uid;
int32_t tid; // table ID
int rowSize; // bytes of a row
int32_t rowSize; // bytes of a row
char * sqlStr; // SQL string
STSchema * pSchema; // pointer to schema array
void * pStream;
......@@ -175,7 +175,7 @@ void cqStop(void *handle) {
pthread_mutex_unlock(&pContext->mutex);
}
void *cqCreate(void *handle, uint64_t uid, int tid, char *sqlStr, STSchema *pSchema) {
void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) {
SCqContext *pContext = handle;
SCqObj *pObj = calloc(sizeof(SCqObj), 1);
......@@ -237,7 +237,7 @@ void cqDrop(void *handle) {
pthread_mutex_unlock(&pContext->mutex);
}
static void doCreateStream(void *param, TAOS_RES *result, int code) {
static void doCreateStream(void *param, TAOS_RES *result, int32_t code) {
SCqObj* pObj = (SCqObj*)param;
SCqContext* pContext = pObj->pContext;
SSqlObj* pSql = (SSqlObj*)result;
......@@ -288,7 +288,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr);
int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
char *buffer = calloc(size, 1);
SWalHead *pHead = (SWalHead *)buffer;
......
......@@ -20,11 +20,11 @@
extern "C" {
#endif
int32_t dnodeInitMnodePeer();
void dnodeCleanupMnodePeer();
int32_t dnodeAllocateMnodePqueue();
void dnodeFreeMnodePqueue();
void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg);
int32_t dnodeInitMPeer();
void dnodeCleanupMPeer();
int32_t dnodeAllocateMPeerQueue();
void dnodeFreeMPeerQueue();
void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg);
#ifdef __cplusplus
}
......
......@@ -20,11 +20,11 @@
extern "C" {
#endif
int32_t dnodeInitMnodeRead();
void dnodeCleanupMnodeRead();
int32_t dnodeAllocateMnodeRqueue();
void dnodeFreeMnodeRqueue();
void dnodeDispatchToMnodeReadQueue(SRpcMsg *rpcMsg);
int32_t dnodeInitMRead();
void dnodeCleanupMRead();
int32_t dnodeAllocMReadQueue();
void dnodeFreeMReadQueue();
void dnodeDispatchToMReadQueue(SRpcMsg *rpcMsg);
#ifdef __cplusplus
}
......
......@@ -20,11 +20,11 @@
extern "C" {
#endif
int32_t dnodeInitMnodeWrite();
void dnodeCleanupMnodeWrite();
int32_t dnodeAllocateMnodeWqueue();
void dnodeFreeMnodeWqueue();
void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg);
int32_t dnodeInitMWrite();
void dnodeCleanupMWrite();
int32_t dnodeAllocMWritequeue();
void dnodeFreeMWritequeue();
void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg);
#ifdef __cplusplus
}
......
......@@ -35,44 +35,44 @@ typedef struct {
typedef struct {
int32_t curNum;
int32_t maxNum;
SMPeerWorker *peerWorker;
SMPeerWorker *worker;
} SMPeerWorkerPool;
static SMPeerWorkerPool tsMPeerPool;
static SMPeerWorkerPool tsMPeerWP;
static taos_qset tsMPeerQset;
static taos_queue tsMPeerQueue;
static void *dnodeProcessMnodePeerQueue(void *param);
static void *dnodeProcessMPeerQueue(void *param);
int32_t dnodeInitMnodePeer() {
int32_t dnodeInitMPeer() {
tsMPeerQset = taosOpenQset();
tsMPeerPool.maxNum = 1;
tsMPeerPool.curNum = 0;
tsMPeerPool.peerWorker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerPool.maxNum);
tsMPeerWP.maxNum = 1;
tsMPeerWP.curNum = 0;
tsMPeerWP.worker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerWP.maxNum);
if (tsMPeerPool.peerWorker == NULL) return -1;
for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
if (tsMPeerWP.worker == NULL) return -1;
for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
SMPeerWorker *pWorker = tsMPeerWP.worker + i;
pWorker->workerId = i;
dDebug("dnode mpeer worker:%d is created", i);
}
dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset);
dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerWP.maxNum, tsMPeerQset);
return 0;
}
void dnodeCleanupMnodePeer() {
for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
void dnodeCleanupMPeer() {
for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
SMPeerWorker *pWorker = tsMPeerWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(tsMPeerQset);
}
dDebug("dnode mpeer worker:%d is closed", i);
}
for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
SMPeerWorker *pWorker = tsMPeerWP.worker + i;
dDebug("dnode mpeer worker:%d start to join", i);
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
......@@ -84,61 +84,60 @@ void dnodeCleanupMnodePeer() {
taosCloseQset(tsMPeerQset);
tsMPeerQset = NULL;
taosTFree(tsMPeerPool.peerWorker);
taosTFree(tsMPeerWP.worker);
}
int32_t dnodeAllocateMnodePqueue() {
int32_t dnodeAllocateMPeerQueue() {
tsMPeerQueue = taosOpenQueue();
if (tsMPeerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
taosAddIntoQset(tsMPeerQset, tsMPeerQueue, NULL);
for (int32_t i = tsMPeerPool.curNum; i < tsMPeerPool.maxNum; ++i) {
SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
for (int32_t i = tsMPeerWP.curNum; i < tsMPeerWP.maxNum; ++i) {
SMPeerWorker *pWorker = tsMPeerWP.worker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodePeerQueue, pWorker) != 0) {
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMPeerQueue, pWorker) != 0) {
dError("failed to create thread to process mpeer queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
tsMPeerPool.curNum = i + 1;
dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerPool.maxNum);
tsMPeerWP.curNum = i + 1;
dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerWP.maxNum);
}
dDebug("dnode mpeer queue:%p is allocated", tsMPeerQueue);
return TSDB_CODE_SUCCESS;
}
void dnodeFreeMnodePqueue() {
void dnodeFreeMPeerQueue() {
dDebug("dnode mpeer queue:%p is freed", tsMPeerQueue);
taosCloseQueue(tsMPeerQueue);
tsMPeerQueue = NULL;
}
void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) {
void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) {
if (!mnodeIsRunning() || tsMPeerQueue == NULL) {
dnodeSendRedirectMsg(pMsg, false);
rpcFreeCont(pMsg->pCont);
return;
} else {
SMnodeMsg *pPeer = mnodeCreateMsg(pMsg);
taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
}
SMnodeMsg *pPeer = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
mnodeCreateMsg(pPeer, pMsg);
taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
rpcFreeCont(pMsg->pCont);
}
static void dnodeFreeMnodePeerMsg(SMnodeMsg *pPeer) {
static void dnodeFreeMPeerMsg(SMnodeMsg *pPeer) {
mnodeCleanupMsg(pPeer);
taosFreeQitem(pPeer);
}
static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
static void dnodeSendRpcMPeerRsp(SMnodeMsg *pPeer, int32_t code) {
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
SRpcMsg rpcRsp = {
......@@ -149,10 +148,10 @@ static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
};
rpcSendResponse(&rpcRsp);
dnodeFreeMnodePeerMsg(pPeer);
dnodeFreeMPeerMsg(pPeer);
}
static void *dnodeProcessMnodePeerQueue(void *param) {
static void *dnodeProcessMPeerQueue(void *param) {
SMnodeMsg *pPeerMsg;
int32_t type;
void * unUsed;
......@@ -165,7 +164,7 @@ static void *dnodeProcessMnodePeerQueue(void *param) {
dDebug("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]);
int32_t code = mnodeProcessPeerReq(pPeerMsg);
dnodeSendRpcMnodePeerRsp(pPeerMsg, code);
dnodeSendRpcMPeerRsp(pPeerMsg, code);
}
return NULL;
......
......@@ -35,46 +35,46 @@ typedef struct {
typedef struct {
int32_t curNum;
int32_t maxNum;
SMReadWorker *readWorker;
SMReadWorker *worker;
} SMReadWorkerPool;
static SMReadWorkerPool tsMReadPool;
static SMReadWorkerPool tsMReadWP;
static taos_qset tsMReadQset;
static taos_queue tsMReadQueue;
static void *dnodeProcessMnodeReadQueue(void *param);
static void *dnodeProcessMReadQueue(void *param);
int32_t dnodeInitMnodeRead() {
int32_t dnodeInitMRead() {
tsMReadQset = taosOpenQset();
tsMReadPool.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
tsMReadPool.maxNum = MAX(2, tsMReadPool.maxNum);
tsMReadPool.maxNum = MIN(4, tsMReadPool.maxNum);
tsMReadPool.curNum = 0;
tsMReadPool.readWorker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadPool.maxNum);
tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum);
tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum);
tsMReadWP.curNum = 0;
tsMReadWP.worker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadWP.maxNum);
if (tsMReadPool.readWorker == NULL) return -1;
for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
SMReadWorker *pWorker = tsMReadPool.readWorker + i;
if (tsMReadWP.worker == NULL) return -1;
for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
SMReadWorker *pWorker = tsMReadWP.worker + i;
pWorker->workerId = i;
dDebug("dnode mread worker:%d is created", i);
}
dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset);
dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadWP.maxNum, tsMReadQset);
return 0;
}
void dnodeCleanupMnodeRead() {
for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
SMReadWorker *pWorker = tsMReadPool.readWorker + i;
void dnodeCleanupMRead() {
for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
SMReadWorker *pWorker = tsMReadWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(tsMReadQset);
}
dDebug("dnode mread worker:%d is closed", i);
}
for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
SMReadWorker *pWorker = tsMReadPool.readWorker + i;
for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
SMReadWorker *pWorker = tsMReadWP.worker + i;
dDebug("dnode mread worker:%d start to join", i);
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
......@@ -86,64 +86,63 @@ void dnodeCleanupMnodeRead() {
taosCloseQset(tsMReadQset);
tsMReadQset = NULL;
free(tsMReadPool.readWorker);
free(tsMReadWP.worker);
}
int32_t dnodeAllocateMnodeRqueue() {
int32_t dnodeAllocMReadQueue() {
tsMReadQueue = taosOpenQueue();
if (tsMReadQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
taosAddIntoQset(tsMReadQset, tsMReadQueue, NULL);
for (int32_t i = tsMReadPool.curNum; i < tsMReadPool.maxNum; ++i) {
SMReadWorker *pWorker = tsMReadPool.readWorker + i;
for (int32_t i = tsMReadWP.curNum; i < tsMReadWP.maxNum; ++i) {
SMReadWorker *pWorker = tsMReadWP.worker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeReadQueue, pWorker) != 0) {
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMReadQueue, pWorker) != 0) {
dError("failed to create thread to process mread queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
tsMReadPool.curNum = i + 1;
dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadPool.maxNum);
tsMReadWP.curNum = i + 1;
dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadWP.maxNum);
}
dDebug("dnode mread queue:%p is allocated", tsMReadQueue);
return TSDB_CODE_SUCCESS;
}
void dnodeFreeMnodeRqueue() {
void dnodeFreeMReadQueue() {
dDebug("dnode mread queue:%p is freed", tsMReadQueue);
taosCloseQueue(tsMReadQueue);
tsMReadQueue = NULL;
}
void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) {
void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) {
if (!mnodeIsRunning() || tsMReadQueue == NULL) {
dnodeSendRedirectMsg(pMsg, true);
rpcFreeCont(pMsg->pCont);
return;
} else {
SMnodeMsg *pRead = mnodeCreateMsg(pMsg);
taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
}
SMnodeMsg *pRead = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
mnodeCreateMsg(pRead, pMsg);
taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
rpcFreeCont(pMsg->pCont);
}
static void dnodeFreeMnodeReadMsg(SMnodeMsg *pRead) {
static void dnodeFreeMReadMsg(SMnodeMsg *pRead) {
mnodeCleanupMsg(pRead);
taosFreeQitem(pRead);
}
static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
static void dnodeSendRpcMReadRsp(SMnodeMsg *pRead, int32_t code) {
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
// may be a auto create req, should put into write queue
dnodeReprocessMnodeWriteMsg(pRead);
dnodeReprocessMWriteMsg(pRead);
return;
}
......@@ -155,23 +154,23 @@ static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
};
rpcSendResponse(&rpcRsp);
dnodeFreeMnodeReadMsg(pRead);
dnodeFreeMReadMsg(pRead);
}
static void *dnodeProcessMnodeReadQueue(void *param) {
SMnodeMsg *pReadMsg;
static void *dnodeProcessMReadQueue(void *param) {
SMnodeMsg *pRead;
int32_t type;
void * unUsed;
while (1) {
if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pReadMsg, &unUsed) == 0) {
if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pRead, &unUsed) == 0) {
dDebug("qset:%p, mnode read got no message from qset, exiting", tsMReadQset);
break;
}
dDebug("%p, msg:%s will be processed in mread queue", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]);
int32_t code = mnodeProcessRead(pReadMsg);
dnodeSendRpcMnodeReadRsp(pReadMsg, code);
dDebug("%p, msg:%s will be processed in mread queue", pRead->rpcMsg.ahandle, taosMsg[pRead->rpcMsg.msgType]);
int32_t code = mnodeProcessRead(pRead);
dnodeSendRpcMReadRsp(pRead, code);
}
return NULL;
......
......@@ -36,45 +36,45 @@ typedef struct {
typedef struct {
int32_t curNum;
int32_t maxNum;
SMWriteWorker *writeWorker;
SMWriteWorker *worker;
} SMWriteWorkerPool;
static SMWriteWorkerPool tsMWritePool;
static SMWriteWorkerPool tsMWriteWP;
static taos_qset tsMWriteQset;
static taos_queue tsMWriteQueue;
extern void * tsDnodeTmr;
static void *dnodeProcessMnodeWriteQueue(void *param);
static void *dnodeProcessMWriteQueue(void *param);
int32_t dnodeInitMnodeWrite() {
int32_t dnodeInitMWrite() {
tsMWriteQset = taosOpenQset();
tsMWritePool.maxNum = 1;
tsMWritePool.curNum = 0;
tsMWritePool.writeWorker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWritePool.maxNum);
tsMWriteWP.maxNum = 1;
tsMWriteWP.curNum = 0;
tsMWriteWP.worker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWriteWP.maxNum);
if (tsMWritePool.writeWorker == NULL) return -1;
for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
if (tsMWriteWP.worker == NULL) return -1;
for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
SMWriteWorker *pWorker = tsMWriteWP.worker + i;
pWorker->workerId = i;
dDebug("dnode mwrite worker:%d is created", i);
}
dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset);
dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWriteWP.maxNum, tsMWriteQset);
return 0;
}
void dnodeCleanupMnodeWrite() {
for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
void dnodeCleanupMWrite() {
for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
SMWriteWorker *pWorker = tsMWriteWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(tsMWriteQset);
}
dDebug("dnode mwrite worker:%d is closed", i);
}
for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
SMWriteWorker *pWorker = tsMWriteWP.worker + i;
dDebug("dnode mwrite worker:%d start to join", i);
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
......@@ -86,58 +86,56 @@ void dnodeCleanupMnodeWrite() {
taosCloseQset(tsMWriteQset);
tsMWriteQset = NULL;
taosTFree(tsMWritePool.writeWorker);
taosTFree(tsMWriteWP.worker);
}
int32_t dnodeAllocateMnodeWqueue() {
int32_t dnodeAllocMWritequeue() {
tsMWriteQueue = taosOpenQueue();
if (tsMWriteQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
taosAddIntoQset(tsMWriteQset, tsMWriteQueue, NULL);
for (int32_t i = tsMWritePool.curNum; i < tsMWritePool.maxNum; ++i) {
SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
for (int32_t i = tsMWriteWP.curNum; i < tsMWriteWP.maxNum; ++i) {
SMWriteWorker *pWorker = tsMWriteWP.worker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeWriteQueue, pWorker) != 0) {
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMWriteQueue, pWorker) != 0) {
dError("failed to create thread to process mwrite queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
tsMWritePool.curNum = i + 1;
dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWritePool.maxNum);
tsMWriteWP.curNum = i + 1;
dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWriteWP.maxNum);
}
dDebug("dnode mwrite queue:%p is allocated", tsMWriteQueue);
return TSDB_CODE_SUCCESS;
}
void dnodeFreeMnodeWqueue() {
void dnodeFreeMWritequeue() {
dDebug("dnode mwrite queue:%p is freed", tsMWriteQueue);
taosCloseQueue(tsMWriteQueue);
tsMWriteQueue = NULL;
}
void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) {
void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
dnodeSendRedirectMsg(pMsg, true);
rpcFreeCont(pMsg->pCont);
return;
} else {
SMnodeMsg *pWrite = mnodeCreateMsg(pMsg);
dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
}
SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
mnodeCreateMsg(pWrite, pMsg);
dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
rpcFreeCont(pMsg->pCont);
}
static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) {
dDebug("app:%p:%p, msg:%s is freed from mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
......@@ -145,12 +143,12 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
taosFreeQitem(pWrite);
}
void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code) {
SMnodeMsg *pWrite = pMsg;
if (pWrite == NULL) return;
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
dnodeReprocessMnodeWriteMsg(pWrite);
dnodeReprocessMWriteMsg(pWrite);
return;
}
......@@ -162,10 +160,10 @@ void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
};
rpcSendResponse(&rpcRsp);
dnodeFreeMnodeWriteMsg(pWrite);
dnodeFreeMWriteMsg(pWrite);
}
static void *dnodeProcessMnodeWriteQueue(void *param) {
static void *dnodeProcessMWriteQueue(void *param) {
SMnodeMsg *pWrite;
int32_t type;
void * unUsed;
......@@ -180,13 +178,13 @@ static void *dnodeProcessMnodeWriteQueue(void *param) {
taosMsg[pWrite->rpcMsg.msgType]);
int32_t code = mnodeProcessWrite(pWrite);
dnodeSendRpcMnodeWriteRsp(pWrite, code);
dnodeSendRpcMWriteRsp(pWrite, code);
}
return NULL;
}
void dnodeReprocessMnodeWriteMsg(void *pMsg) {
void dnodeReprocessMWriteMsg(void *pMsg) {
SMnodeMsg *pWrite = pMsg;
if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
......@@ -194,7 +192,7 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
dnodeSendRedirectMsg(pMsg, true);
dnodeFreeMnodeWriteMsg(pWrite);
dnodeFreeMWriteMsg(pWrite);
} else {
dDebug("app:%p:%p, msg:%s is reput into mwrite queue:%p, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry);
......@@ -203,12 +201,12 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
}
}
static void dnodeDoDelayReprocessMnodeWriteMsg(void *param, void *tmrId) {
dnodeReprocessMnodeWriteMsg(param);
static void dnodeDoDelayReprocessMWriteMsg(void *param, void *tmrId) {
dnodeReprocessMWriteMsg(param);
}
void dnodeDelayReprocessMnodeWriteMsg(void *pMsg) {
void dnodeDelayReprocessMWriteMsg(void *pMsg) {
SMnodeMsg *mnodeMsg = pMsg;
void *unUsed = NULL;
taosTmrReset(dnodeDoDelayReprocessMnodeWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
taosTmrReset(dnodeDoDelayReprocessMWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
}
......@@ -37,11 +37,11 @@
#include "dnodeShell.h"
#include "dnodeTelemetry.h"
static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED;
static int32_t dnodeInitStorage();
static void dnodeCleanupStorage();
static void dnodeSetRunStatus(SDnodeRunStatus status);
static void dnodeSetRunStatus(SRunStatus status);
static void dnodeCheckDataDirOpenned(char *dir);
static int32_t dnodeInitComponents();
static void dnodeCleanupComponents(int32_t stepId);
......@@ -63,9 +63,9 @@ static const SDnodeComponent tsDnodeComponents[] = {
{"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
{"vread", dnodeInitVRead, dnodeCleanupVRead},
{"vwrite", dnodeInitVWrite, dnodeCleanupVWrite},
{"mread", dnodeInitMnodeRead, dnodeCleanupMnodeRead},
{"mwrite", dnodeInitMnodeWrite, dnodeCleanupMnodeWrite},
{"mpeer", dnodeInitMnodePeer, dnodeCleanupMnodePeer},
{"mread", dnodeInitMRead, dnodeCleanupMRead},
{"mwrite", dnodeInitMWrite, dnodeCleanupMWrite},
{"mpeer", dnodeInitMPeer, dnodeCleanupMPeer},
{"client", dnodeInitClient, dnodeCleanupClient},
{"server", dnodeInitServer, dnodeCleanupServer},
{"mgmt", dnodeInitMgmt, dnodeCleanupMgmt},
......@@ -104,7 +104,7 @@ static int32_t dnodeInitComponents() {
}
int32_t dnodeInitSystem() {
dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE);
dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE);
tscEmbedded = 1;
taosBlockSIGPIPE();
taosResolveCRC();
......@@ -137,7 +137,7 @@ int32_t dnodeInitSystem() {
}
dnodeStartModules();
dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING);
dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING);
dInfo("TDengine is initialized successfully");
......@@ -145,20 +145,20 @@ int32_t dnodeInitSystem() {
}
void dnodeCleanUpSystem() {
if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) {
dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED);
dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1);
taos_cleanup();
taosCloseLog();
}
}
SDnodeRunStatus dnodeGetRunStatus() {
return tsDnodeRunStatus;
SRunStatus dnodeGetRunStatus() {
return tsRunStatus;
}
static void dnodeSetRunStatus(SDnodeRunStatus status) {
tsDnodeRunStatus = status;
static void dnodeSetRunStatus(SRunStatus status) {
tsRunStatus = status;
}
static void dnodeCheckDataDirOpenned(char *dir) {
......
......@@ -47,6 +47,11 @@ typedef struct {
int32_t * vnodeList;
} SOpenVnodeThread;
typedef struct {
SRpcMsg rpcMsg;
char pCont[];
} SMgmtMsg;
void * tsDnodeTmr = NULL;
static void * tsStatusTimer = NULL;
static uint32_t tsRebootTime;
......@@ -172,38 +177,46 @@ void dnodeCleanupMgmt() {
vnodeCleanupResources();
}
void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
void *item;
static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
int32_t size = sizeof(SMgmtMsg) + pMsg->contLen;
SMgmtMsg *pMgmt = taosAllocateQitem(size);
if (pMgmt == NULL) {
return TSDB_CODE_DND_OUT_OF_MEMORY;
}
item = taosAllocateQitem(sizeof(SRpcMsg));
if (item) {
memcpy(item, pMsg, sizeof(SRpcMsg));
taosWriteQitem(tsMgmtQueue, 1, item);
} else {
SRpcMsg rsp = {
.handle = pMsg->handle,
.pCont = NULL,
.code = TSDB_CODE_DND_OUT_OF_MEMORY
};
pMgmt->rpcMsg = *pMsg;
pMgmt->rpcMsg.pCont = pMgmt->pCont;
memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt);
return TSDB_CODE_SUCCESS;
}
void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
int32_t code = dnodeWriteToMgmtQueue(pMsg);
if (code != TSDB_CODE_SUCCESS) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
rpcSendResponse(&rsp);
rpcFreeCont(pMsg->pCont);
}
rpcFreeCont(pMsg->pCont);
}
static void *dnodeProcessMgmtQueue(void *param) {
SRpcMsg *pMsg;
SRpcMsg rsp = {0};
int type;
void * handle;
SMgmtMsg *pMgmt;
SRpcMsg * pMsg;
SRpcMsg rsp = {0};
int32_t qtype;
void * handle;
while (1) {
if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) {
if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) {
dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset);
break;
}
dDebug("%p, msg:%s will be processed", pMsg->ahandle, taosMsg[pMsg->msgType]);
pMsg = &pMgmt->rpcMsg;
dDebug("%p, msg:%p:%s will be processed", pMsg->ahandle, pMgmt, taosMsg[pMsg->msgType]);
if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
} else {
......@@ -211,10 +224,9 @@ static void *dnodeProcessMgmtQueue(void *param) {
}
rsp.handle = pMsg->handle;
rsp.pCont = NULL;
rsp.pCont = NULL;
rpcSendResponse(&rsp);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
......
......@@ -19,6 +19,7 @@
* to dnode. All theses messages are handled from here
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosmsg.h"
#include "tglobal.h"
......@@ -34,8 +35,8 @@ static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *);
static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet);
static void *tsDnodeServerRpc = NULL;
static void *tsDnodeClientRpc = NULL;
static void *tsServerRpc = NULL;
static void *tsClientRpc = NULL;
int32_t dnodeInitServer() {
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVWriteQueue;
......@@ -50,11 +51,11 @@ int32_t dnodeInitServer() {
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMnodePeerQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMnodePeerQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMnodePeerQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMnodePeerQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMnodePeerQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMPeerQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMPeerQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMPeerQueue;
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
......@@ -66,8 +67,8 @@ int32_t dnodeInitServer() {
rpcInit.connType = TAOS_CONN_SERVER;
rpcInit.idleTime = tsShellActivityTimer * 1000;
tsDnodeServerRpc = rpcOpen(&rpcInit);
if (tsDnodeServerRpc == NULL) {
tsServerRpc = rpcOpen(&rpcInit);
if (tsServerRpc == NULL) {
dError("failed to init inter-dnodes RPC server");
return -1;
}
......@@ -77,9 +78,9 @@ int32_t dnodeInitServer() {
}
void dnodeCleanupServer() {
if (tsDnodeServerRpc) {
rpcClose(tsDnodeServerRpc);
tsDnodeServerRpc = NULL;
if (tsServerRpc) {
rpcClose(tsServerRpc);
tsServerRpc = NULL;
dInfo("inter-dnodes RPC server is closed");
}
}
......@@ -93,7 +94,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
if (pMsg->pCont == NULL) return;
if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
rspMsg.code = TSDB_CODE_APP_NOT_READY;
rpcSendResponse(&rspMsg);
rpcFreeCont(pMsg->pCont);
......@@ -131,8 +132,8 @@ int32_t dnodeInitClient() {
rpcInit.ckey = "key";
rpcInit.secret = secret;
tsDnodeClientRpc = rpcOpen(&rpcInit);
if (tsDnodeClientRpc == NULL) {
tsClientRpc = rpcOpen(&rpcInit);
if (tsClientRpc == NULL) {
dError("failed to init mnode rpc client");
return -1;
}
......@@ -142,9 +143,9 @@ int32_t dnodeInitClient() {
}
void dnodeCleanupClient() {
if (tsDnodeClientRpc) {
rpcClose(tsDnodeClientRpc);
tsDnodeClientRpc = NULL;
if (tsClientRpc) {
rpcClose(tsClientRpc);
tsClientRpc = NULL;
dInfo("dnode inter-dnodes rpc client is closed");
}
}
......@@ -168,15 +169,15 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
}
void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg);
rpcSendRequest(tsClientRpc, epSet, rpcMsg);
}
void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
SRpcEpSet epSet = {0};
dnodeGetEpSetForPeer(&epSet);
rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp);
rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp);
}
void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) {
rpcSendRecv(tsDnodeClientRpc, epSet, rpcMsg, rpcRsp);
rpcSendRecv(tsClientRpc, epSet, rpcMsg, rpcRsp);
}
\ No newline at end of file
......@@ -33,9 +33,9 @@
static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *);
static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey);
static void * tsDnodeShellRpc = NULL;
static int32_t tsDnodeQueryReqNum = 0;
static int32_t tsDnodeSubmitReqNum = 0;
static void * tsShellRpc = NULL;
static int32_t tsQueryReqNum = 0;
static int32_t tsSubmitReqNum = 0;
int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVWriteQueue;
......@@ -44,35 +44,35 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue;
// the following message shall be treated as mnode write
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMnodeWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue;
// the following message shall be treated as mnode query
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMnodeReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMnodeReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMnodeReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMnodeReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMnodeReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMnodeReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMnodeReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMnodeReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue;
int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore;
numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
......@@ -91,8 +91,8 @@ int32_t dnodeInitShell() {
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.afp = dnodeRetrieveUserAuthInfo;
tsDnodeShellRpc = rpcOpen(&rpcInit);
if (tsDnodeShellRpc == NULL) {
tsShellRpc = rpcOpen(&rpcInit);
if (tsShellRpc == NULL) {
dError("failed to init shell rpc server");
return -1;
}
......@@ -102,13 +102,13 @@ int32_t dnodeInitShell() {
}
void dnodeCleanupShell() {
if (tsDnodeShellRpc) {
rpcClose(tsDnodeShellRpc);
tsDnodeShellRpc = NULL;
if (tsShellRpc) {
rpcClose(tsShellRpc);
tsShellRpc = NULL;
}
}
void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SRpcMsg rpcMsg = {
.handle = pMsg->handle,
.pCont = NULL,
......@@ -117,7 +117,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
if (pMsg->pCont == NULL) return;
if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
rpcMsg.code = TSDB_CODE_APP_NOT_READY;
rpcSendResponse(&rpcMsg);
......@@ -126,9 +126,9 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
}
if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) {
atomic_fetch_add_32(&tsDnodeQueryReqNum, 1);
atomic_fetch_add_32(&tsQueryReqNum, 1);
} else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
atomic_fetch_add_32(&tsDnodeSubmitReqNum, 1);
atomic_fetch_add_32(&tsSubmitReqNum, 1);
} else {}
if ( dnodeProcessShellMsgFp[pMsg->msgType] ) {
......@@ -211,12 +211,12 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) {
}
}
SDnodeStatisInfo dnodeGetStatisInfo() {
SDnodeStatisInfo info = {0};
if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_RUNING) {
SStatisInfo dnodeGetStatisInfo() {
SStatisInfo info = {0};
if (dnodeGetRunStatus() == TSDB_RUN_STATUS_RUNING) {
info.httpReqNum = httpGetReqCount();
info.queryReqNum = atomic_exchange_32(&tsDnodeQueryReqNum, 0);
info.submitReqNum = atomic_exchange_32(&tsDnodeSubmitReqNum, 0);
info.queryReqNum = atomic_exchange_32(&tsQueryReqNum, 0);
info.submitReqNum = atomic_exchange_32(&tsSubmitReqNum, 0);
}
return info;
......
......@@ -113,7 +113,7 @@ void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) {
void *dnodeAllocVWriteQueue(void *pVnode) {
pthread_mutex_lock(&tsVWriteWP.mutex);
SVWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId;
void *queue = taosOpenQueue();
taos_queue *queue = taosOpenQueue();
if (queue == NULL) {
pthread_mutex_unlock(&tsVWriteWP.mutex);
return NULL;
......
......@@ -27,16 +27,16 @@ typedef struct {
int32_t queryReqNum;
int32_t submitReqNum;
int32_t httpReqNum;
} SDnodeStatisInfo;
} SStatisInfo;
typedef enum {
TSDB_DNODE_RUN_STATUS_INITIALIZE,
TSDB_DNODE_RUN_STATUS_RUNING,
TSDB_DNODE_RUN_STATUS_STOPPED
} SDnodeRunStatus;
TSDB_RUN_STATUS_INITIALIZE,
TSDB_RUN_STATUS_RUNING,
TSDB_RUN_STATUS_STOPPED
} SRunStatus;
SDnodeRunStatus dnodeGetRunStatus();
SDnodeStatisInfo dnodeGetStatisInfo();
SRunStatus dnodeGetRunStatus();
SStatisInfo dnodeGetStatisInfo();
bool dnodeIsFirstDeploy();
bool dnodeIsMasterEp(char *ep);
......@@ -59,15 +59,15 @@ void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code);
void *dnodeAllocVReadQueue(void *pVnode);
void dnodeFreeVReadQueue(void *rqueue);
int32_t dnodeAllocateMnodePqueue();
void dnodeFreeMnodePqueue();
int32_t dnodeAllocateMnodeRqueue();
void dnodeFreeMnodeRqueue();
int32_t dnodeAllocateMnodeWqueue();
void dnodeFreeMnodeWqueue();
void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code);
void dnodeReprocessMnodeWriteMsg(void *pMsg);
void dnodeDelayReprocessMnodeWriteMsg(void *pMsg);
int32_t dnodeAllocateMPeerQueue();
void dnodeFreeMPeerQueue();
int32_t dnodeAllocMReadQueue();
void dnodeFreeMReadQueue();
int32_t dnodeAllocMWritequeue();
void dnodeFreeMWritequeue();
void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code);
void dnodeReprocessMWriteMsg(void *pMsg);
void dnodeDelayReprocessMWriteMsg(void *pMsg);
void dnodeSendStatusMsgToMnode();
......
......@@ -35,7 +35,13 @@ typedef struct {
} SMnodeRsp;
typedef struct SMnodeMsg {
SRpcMsg rpcMsg;
struct SAcctObj * pAcct;
struct SDnodeObj *pDnode;
struct SUserObj * pUser;
struct SDbObj * pDb;
struct SVgObj * pVgroup;
struct STableObj *pTable;
struct SSTableObj*pSTable;
SMnodeRsp rpcRsp;
int8_t received;
int8_t successed;
......@@ -43,16 +49,11 @@ typedef struct SMnodeMsg {
int8_t retry;
int32_t code;
void * pObj;
struct SAcctObj * pAcct;
struct SDnodeObj *pDnode;
struct SUserObj * pUser;
struct SDbObj * pDb;
struct SVgObj * pVgroup;
struct STableObj *pTable;
struct SSuperTableObj *pSTable;
SRpcMsg rpcMsg;
char pCont[];
} SMnodeMsg;
void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg);
void * mnodeCreateMsg(SRpcMsg *pRpcMsg);
int32_t mnodeInitMsg(SMnodeMsg *pMsg);
void mnodeCleanupMsg(SMnodeMsg *pMsg);
......
......@@ -429,42 +429,43 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_PORT_DNODEDNODE 5
#define TSDB_PORT_SYNC 10
#define TSDB_PORT_HTTP 11
#define TSDB_PORT_ARBITRATOR 12
#define TSDB_PORT_ARBITRATOR 12
#define TAOS_QTYPE_RPC 0
#define TAOS_QTYPE_FWD 1
#define TAOS_QTYPE_WAL 2
#define TAOS_QTYPE_CQ 3
#define TAOS_QTYPE_QUERY 4
typedef enum {
TAOS_QTYPE_RPC = 0,
TAOS_QTYPE_FWD = 1,
TAOS_QTYPE_WAL = 2,
TAOS_QTYPE_CQ = 3,
TAOS_QTYPE_QUERY = 4
} EQType;
typedef enum {
TSDB_SUPER_TABLE = 0, // super table
TSDB_CHILD_TABLE = 1, // table created from super table
TSDB_NORMAL_TABLE = 2, // ordinary table
TSDB_STREAM_TABLE = 3, // table created from stream computing
TSDB_TABLE_MAX = 4
TSDB_SUPER_TABLE = 0, // super table
TSDB_CHILD_TABLE = 1, // table created from super table
TSDB_NORMAL_TABLE = 2, // ordinary table
TSDB_STREAM_TABLE = 3, // table created from stream computing
TSDB_TABLE_MAX = 4
} ETableType;
typedef enum {
TSDB_MOD_MNODE,
TSDB_MOD_HTTP,
TSDB_MOD_MONITOR,
TSDB_MOD_MQTT,
TSDB_MOD_MAX
TSDB_MOD_MNODE = 0,
TSDB_MOD_HTTP = 1,
TSDB_MOD_MONITOR = 2,
TSDB_MOD_MQTT = 3,
TSDB_MOD_MAX = 4
} EModuleType;
typedef enum {
TSDB_CHECK_ITEM_NETWORK,
TSDB_CHECK_ITEM_MEM,
TSDB_CHECK_ITEM_CPU,
TSDB_CHECK_ITEM_DISK,
TSDB_CHECK_ITEM_OS,
TSDB_CHECK_ITEM_ACCESS,
TSDB_CHECK_ITEM_VERSION,
TSDB_CHECK_ITEM_DATAFILE,
TSDB_CHECK_ITEM_MAX
} ECheckItemType;
typedef enum {
TSDB_CHECK_ITEM_NETWORK,
TSDB_CHECK_ITEM_MEM,
TSDB_CHECK_ITEM_CPU,
TSDB_CHECK_ITEM_DISK,
TSDB_CHECK_ITEM_OS,
TSDB_CHECK_ITEM_ACCESS,
TSDB_CHECK_ITEM_VERSION,
TSDB_CHECK_ITEM_DATAFILE,
TSDB_CHECK_ITEM_MAX
} ECheckItemType;
#ifdef __cplusplus
}
......
......@@ -24,7 +24,7 @@ extern "C" {
typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
typedef struct {
int vgId;
int32_t vgId;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_NAME_LEN];
......@@ -42,12 +42,12 @@ void cqStart(void *handle);
void cqStop(void *handle);
// cqCreate is called by TSDB to start an instance of CQ
void *cqCreate(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema);
// cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate
void cqDrop(void *handle);
extern int cqDebugFlag;
extern int32_t cqDebugFlag;
#ifdef __cplusplus
......
......@@ -83,13 +83,13 @@ void rpcClose(void *);
void *rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont);
void *rpcReallocCont(void *ptr, int contLen);
void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
int64_t rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
void rpcSendResponse(const SRpcMsg *pMsg);
void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen);
void rpcCancelRequest(void *pContext);
void rpcCancelRequest(int64_t rid);
#ifdef __cplusplus
}
......
......@@ -164,6 +164,12 @@ typedef struct STsdbQueryCond {
SColumnInfo *colList;
} STsdbQueryCond;
typedef struct SMemRef {
int32_t ref;
void *mem;
void *imem;
} SMemRef;
typedef struct SDataBlockInfo {
STimeWindow window;
int32_t rows;
......@@ -193,7 +199,7 @@ typedef struct {
* @param qinfo query info handle from query processor
* @return
*/
TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo);
TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef);
/**
* Get the last row of the given query time window for all the tables in STableGroupInfo object.
......@@ -205,7 +211,7 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
* @param tableInfo table list.
* @return
*/
TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo);
TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef);
/**
* get the queried table object list
......@@ -223,7 +229,7 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
* @return
*/
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
void *qinfo);
void *qinfo, SMemRef* pRef);
/**
* move to next block if exists
......
......@@ -106,13 +106,13 @@ typedef void* tsync_h;
int32_t syncInit();
void syncCleanUp();
tsync_h syncStart(const SSyncInfo *);
void syncStop(tsync_h shandle);
int32_t syncReconfig(tsync_h shandle, const SSyncCfg *);
int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype);
void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code);
void syncRecover(tsync_h shandle); // recover from other nodes:
int syncGetNodesRole(tsync_h shandle, SNodesRole *);
int64_t syncStart(const SSyncInfo *);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg *);
int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype);
void syncConfirmForward(int64_t rid, uint64_t version, int32_t code);
void syncRecover(int64_t rid); // recover from other nodes:
int syncGetNodesRole(int64_t rid, SNodesRole *);
extern char *syncRole[];
......
......@@ -25,9 +25,15 @@ typedef enum {
TAOS_WAL_FSYNC = 2
} EWalType;
typedef enum {
TAOS_WAL_NOT_KEEP = 0,
TAOS_WAL_KEEP = 1
} EWalKeep;
typedef struct {
int8_t msgType;
int8_t reserved[3];
int8_t sver;
int8_t reserved[2];
int32_t len;
uint64_t version;
uint32_t signature;
......@@ -36,11 +42,10 @@ typedef struct {
} SWalHead;
typedef struct {
int32_t vgId;
int32_t fsyncPeriod; // millisecond
int8_t walLevel; // wal level
int8_t wals; // number of WAL files;
int8_t keep; // keep the wal file when closed
int32_t vgId;
int32_t fsyncPeriod; // millisecond
EWalType walLevel; // wal level
EWalKeep keep; // keep the wal file when closed
} SWalCfg;
typedef void * twalh; // WAL HANDLE
......@@ -58,7 +63,7 @@ int32_t walWrite(twalh, SWalHead *);
void walFsync(twalh, bool forceFsync);
int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp);
int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId);
int64_t walGetVersion(twalh);
uint64_t walGetVersion(twalh);
#ifdef __cplusplus
}
......
......@@ -793,9 +793,9 @@ int main(int argc, char *argv[]) {
(ntables * nrecords_per_table) / (t * nrecords_per_request),
t * 1000);
printf("Spent %.4f seconds to insert %d records with %d record(s) per request: %.2f records/second\n",
t, ntables * nrecords_per_table, nrecords_per_request,
ntables * nrecords_per_table / t);
printf("Spent %.4f seconds to insert %lld records with %d record(s) per request: %.2f records/second\n",
t, (long long int)ntables * nrecords_per_table, nrecords_per_request,
((long long int)ntables * nrecords_per_table) / t);
for (int i = 0; i < threads; i++) {
info *t_info = infos + i;
......
......@@ -89,7 +89,7 @@ typedef struct STableObj {
int8_t type;
} STableObj;
typedef struct SSuperTableObj {
typedef struct SSTableObj {
STableObj info;
int8_t reserved0[9]; // for fill struct STableObj to 4byte align
int16_t nextColId;
......@@ -104,7 +104,7 @@ typedef struct SSuperTableObj {
int32_t numOfTables;
SSchema * schema;
void * vgHash;
} SSuperTableObj;
} SSTableObj;
typedef struct {
STableObj info;
......@@ -122,8 +122,8 @@ typedef struct {
int32_t refCount;
char* sql; //used by normal table
SSchema* schema; //used by normal table
SSuperTableObj *superTable;
} SChildTableObj;
SSTableObj*superTable;
} SCTableObj;
typedef struct {
int32_t dnodeId;
......
......@@ -29,8 +29,8 @@ int64_t mnodeGetChildTableNum();
void * mnodeGetTable(char *tableId);
void mnodeIncTableRef(void *pTable);
void mnodeDecTableRef(void *pTable);
void * mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable);
void * mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
void * mnodeGetNextChildTable(void *pIter, SCTableObj **pTable);
void * mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable);
void mnodeDropAllChildTables(SDbObj *pDropDb);
void mnodeDropAllSuperTables(SDbObj *pDropDb);
void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup);
......
......@@ -43,8 +43,8 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle);
void mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle);
int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid);
void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable);
void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable);
void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle);
void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
void mnodeSendAlterVgroupMsg(SVgObj *pVgroup);
......
......@@ -18,7 +18,7 @@
#include "taosmsg.h"
#include "taoserror.h"
#include "trpc.h"
#include "tcache.h"
#include "tqueue.h"
#include "mnode.h"
#include "dnode.h"
#include "mnodeDef.h"
......@@ -34,8 +34,15 @@
#include "mnodeUser.h"
#include "mnodeVgroup.h"
void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg) {
pMsg->rpcMsg = *rpcMsg;
void *mnodeCreateMsg(SRpcMsg *pRpcMsg) {
int32_t size = sizeof(SMnodeMsg) + pRpcMsg->contLen;
SMnodeMsg *pMsg = taosAllocateQitem(size);
pMsg->rpcMsg = *pRpcMsg;
pMsg->rpcMsg.pCont = pMsg->pCont;
memcpy(pMsg->pCont, pRpcMsg->pCont, pRpcMsg->contLen);
return pMsg;
}
int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
......@@ -54,7 +61,9 @@ int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
void mnodeCleanupMsg(SMnodeMsg *pMsg) {
if (pMsg != NULL) {
if (pMsg->rpcMsg.pCont) rpcFreeCont(pMsg->rpcMsg.pCont);
if (pMsg->rpcMsg.pCont != pMsg->pCont) {
tfree(pMsg->rpcMsg.pCont);
}
if (pMsg->pUser) mnodeDecUserRef(pMsg->pUser);
if (pMsg->pDb) mnodeDecDbRef(pMsg->pDb);
if (pMsg->pVgroup) mnodeDecVgroupRef(pMsg->pVgroup);
......
......@@ -96,9 +96,9 @@ int32_t mnodeStartSystem() {
return -1;
}
dnodeAllocateMnodeWqueue();
dnodeAllocateMnodeRqueue();
dnodeAllocateMnodePqueue();
dnodeAllocMWritequeue();
dnodeAllocMReadQueue();
dnodeAllocateMPeerQueue();
if (mnodeInitComponents() != 0) {
return -1;
......@@ -127,9 +127,9 @@ void mnodeCleanupSystem() {
mInfo("starting to clean up mnode");
tsMgmtIsRunning = false;
dnodeFreeMnodeWqueue();
dnodeFreeMnodeRqueue();
dnodeFreeMnodePqueue();
dnodeFreeMWritequeue();
dnodeFreeMReadQueue();
dnodeFreeMPeerQueue();
mnodeCleanupTimer();
mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1);
......
......@@ -72,7 +72,7 @@ typedef struct {
ESyncRole role;
ESdbStatus status;
int64_t version;
void * sync;
int64_t sync;
void * wal;
SSyncCfg cfg;
int32_t numOfTables;
......@@ -175,7 +175,7 @@ static void *sdbGetTableFromId(int32_t tableId) {
}
static int32_t sdbInitWal() {
SWalCfg walCfg = {.vgId = 1, .walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0};
SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0};
char temp[TSDB_FILENAME_LEN];
sprintf(temp, "%s/wal", tsMnodeDir);
tsSdbObj.wal = walOpen(temp, &walCfg);
......@@ -212,7 +212,7 @@ static void sdbRestoreTables() {
}
void sdbUpdateMnodeRoles() {
if (tsSdbObj.sync == NULL) return;
if (tsSdbObj.sync <= 0) return;
SNodesRole roles = {0};
syncGetNodesRole(tsSdbObj.sync, &roles);
......@@ -295,7 +295,7 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
if (pOper->writeCb != NULL) {
pOper->retCode = (*pOper->writeCb)(pMsg, pOper->retCode);
}
dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode);
dnodeSendRpcMWriteRsp(pMsg, pOper->retCode);
// if ahandle, means this func is called by sdb write
if (ahandle == NULL) {
......@@ -433,7 +433,7 @@ void sdbCleanUp() {
if (tsSdbObj.sync) {
syncStop(tsSdbObj.sync);
tsSdbObj.sync = NULL;
tsSdbObj.sync = -1;
}
if (tsSdbObj.wal) {
......@@ -1043,7 +1043,7 @@ void sdbFreeWritequeue() {
int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) {
SWalHead *pHead = data;
int32_t size = sizeof(SWalHead) + pHead->len;
SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
SWalHead *pWal = taosAllocateQitem(size);
memcpy(pWal, pHead, size);
taosWriteQitem(tsSdbWriteQueue, qtype, pWal);
......
此差异已折叠。
......@@ -529,7 +529,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
(void)sdbUpdateRow(&desc);
dnodeReprocessMnodeWriteMsg(pMsg);
dnodeReprocessMWriteMsg(pMsg);
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
// if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_READY) {
// mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
......@@ -537,7 +537,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
// pVgroup->status = TAOS_VG_STATUS_READY;
// SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
// (void)sdbUpdateRow(&desc);
// dnodeReprocessMnodeWriteMsg(pMsg);
// dnodeReprocessMWriteMsg(pMsg);
// return TSDB_CODE_MND_ACTION_IN_PROGRESS;
// } else {
// mError("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d, but vgroup is dropping", pMsg->rpcMsg.ahandle,
......@@ -694,7 +694,7 @@ static bool mnodeFilterVgroups(SVgObj *pVgroup, STableObj *pTable) {
return true;
}
SChildTableObj *pCTable = (SChildTableObj *)pTable;
SCTableObj *pCTable = (SCTableObj *)pTable;
if (pVgroup->vgId == pCTable->vgId) {
return true;
} else {
......@@ -791,7 +791,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
return numOfRows;
}
void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable) {
int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool);
if (pTable->tid > idPoolSize) {
mnodeAllocVgroupIdPool(pVgroup);
......@@ -807,7 +807,7 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
}
}
void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable) {
if (pTable->tid >= 1) {
taosFreeId(pVgroup->idPool, pTable->tid);
pVgroup->numOfTables--;
......@@ -970,7 +970,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mnodeMsg->pVgroup = NULL;
mnodeDestroyVgroup(pVgroup);
dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
dnodeSendRpcMWriteRsp(mnodeMsg, code);
}
} else {
SSdbOper oper = {
......@@ -979,7 +979,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
.pObj = pVgroup
};
sdbDeleteRow(&oper);
dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code);
dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code);
}
}
......@@ -1041,7 +1041,7 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
code = TSDB_CODE_MND_SDB_ERROR;
}
dnodeReprocessMnodeWriteMsg(mnodeMsg);
dnodeReprocessMWriteMsg(mnodeMsg);
}
static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {
......
......@@ -49,7 +49,7 @@ static taos_queue tsHttpQueue;
void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t numOfRows, void (*fp)(void *param, void *result, int32_t numOfRows)) {
if (tsHttpQueue != NULL) {
SHttpResult *pMsg = (SHttpResult *)taosAllocateQitem(sizeof(SHttpResult));
SHttpResult *pMsg = taosAllocateQitem(sizeof(SHttpResult));
pMsg->param = param;
pMsg->result = result;
pMsg->numOfRows = numOfRows;
......
......@@ -27,12 +27,12 @@
#include "monitor.h"
#include "taoserror.h"
#define monitorFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
#define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
#define monitorWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
#define monitorInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }}
#define monitorDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
#define monitorTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
#define mnFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
#define mnError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
#define mnWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
#define mnInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }}
#define mnDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
#define mnTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
#define SQL_LENGTH 1030
#define LOG_LEN_STR 100
......@@ -91,12 +91,12 @@ int32_t monitorInitSystem() {
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&tsMonitor.thread, &thAttr, monitorThreadFunc, NULL)) {
monitorError("failed to create thread to for monitor module, reason:%s", strerror(errno));
mnError("failed to create thread to for monitor module, reason:%s", strerror(errno));
return -1;
}
pthread_attr_destroy(&thAttr);
monitorDebug("monitor thread is launched");
mnDebug("monitor thread is launched");
monitorStartSystemFp = monitorStartSystem;
monitorStopSystemFp = monitorStopSystem;
......@@ -107,12 +107,12 @@ int32_t monitorStartSystem() {
taos_init();
tsMonitor.start = 1;
monitorExecuteSQLFp = monitorExecuteSQL;
monitorInfo("monitor module start");
mnInfo("monitor module start");
return 0;
}
static void *monitorThreadFunc(void *param) {
monitorDebug("starting to initialize monitor module ...");
mnDebug("starting to initialize monitor module ...");
while (1) {
static int32_t accessTimes = 0;
......@@ -121,7 +121,7 @@ static void *monitorThreadFunc(void *param) {
if (tsMonitor.quiting) {
tsMonitor.state = MON_STATE_NOT_INIT;
monitorInfo("monitor thread will quit, for taosd is quiting");
mnInfo("monitor thread will quit, for taosd is quiting");
break;
} else {
taosGetDisk();
......@@ -132,7 +132,7 @@ static void *monitorThreadFunc(void *param) {
}
if (dnodeGetDnodeId() <= 0) {
monitorDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
mnDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
continue;
}
......@@ -140,10 +140,10 @@ static void *monitorThreadFunc(void *param) {
tsMonitor.state = MON_STATE_NOT_INIT;
tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0);
if (tsMonitor.conn == NULL) {
monitorError("failed to connect to database, reason:%s", tstrerror(terrno));
mnError("failed to connect to database, reason:%s", tstrerror(terrno));
continue;
} else {
monitorDebug("connect to database success");
mnDebug("connect to database success");
}
}
......@@ -155,10 +155,10 @@ static void *monitorThreadFunc(void *param) {
taos_free_result(res);
if (code != 0) {
monitorError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
mnError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
break;
} else {
monitorDebug("successfully to exec sql:%s", tsMonitor.sql);
mnDebug("successfully to exec sql:%s", tsMonitor.sql);
}
}
......@@ -174,7 +174,7 @@ static void *monitorThreadFunc(void *param) {
}
}
monitorInfo("monitor thread is stopped");
mnInfo("monitor thread is stopped");
return NULL;
}
......@@ -238,7 +238,7 @@ void monitorStopSystem() {
tsMonitor.start = 0;
tsMonitor.state = MON_STATE_NOT_INIT;
monitorExecuteSQLFp = NULL;
monitorInfo("monitor module stopped");
mnInfo("monitor module stopped");
}
void monitorCleanUpSystem() {
......@@ -249,7 +249,7 @@ void monitorCleanUpSystem() {
taos_close(tsMonitor.conn);
tsMonitor.conn = NULL;
}
monitorInfo("monitor module is cleaned up");
mnInfo("monitor module is cleaned up");
}
// unit is MB
......@@ -257,13 +257,13 @@ static int32_t monitorBuildMemorySql(char *sql) {
float sysMemoryUsedMB = 0;
bool suc = taosGetSysMemory(&sysMemoryUsedMB);
if (!suc) {
monitorDebug("failed to get sys memory info");
mnDebug("failed to get sys memory info");
}
float procMemoryUsedMB = 0;
suc = taosGetProcMemory(&procMemoryUsedMB);
if (!suc) {
monitorDebug("failed to get proc memory info");
mnDebug("failed to get proc memory info");
}
return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB);
......@@ -274,7 +274,7 @@ static int32_t monitorBuildCpuSql(char *sql) {
float sysCpuUsage = 0, procCpuUsage = 0;
bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
if (!suc) {
monitorDebug("failed to get cpu usage");
mnDebug("failed to get cpu usage");
}
if (sysCpuUsage <= procCpuUsage) {
......@@ -294,14 +294,14 @@ static int32_t monitorBuildBandSql(char *sql) {
float bandSpeedKb = 0;
bool suc = taosGetBandSpeed(&bandSpeedKb);
if (!suc) {
monitorDebug("failed to get bandwidth speed");
mnDebug("failed to get bandwidth speed");
}
return sprintf(sql, ", %f", bandSpeedKb);
}
static int32_t monitorBuildReqSql(char *sql) {
SDnodeStatisInfo info = dnodeGetStatisInfo();
SStatisInfo info = dnodeGetStatisInfo();
return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum);
}
......@@ -309,7 +309,7 @@ static int32_t monitorBuildIoSql(char *sql) {
float readKB = 0, writeKB = 0;
bool suc = taosGetProcIO(&readKB, &writeKB);
if (!suc) {
monitorDebug("failed to get io info");
mnDebug("failed to get io info");
}
return sprintf(sql, ", %f, %f", readKB, writeKB);
......@@ -332,19 +332,19 @@ static void monitorSaveSystemInfo() {
taos_free_result(res);
if (code != 0) {
monitorError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
mnError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
} else {
monitorDebug("successfully to save system info, sql:%s", tsMonitor.sql);
mnDebug("successfully to save system info, sql:%s", tsMonitor.sql);
}
}
static void montiorExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
int32_t c = taos_errno(result);
if (c != TSDB_CODE_SUCCESS) {
monitorError("save %s failed, reason:%s", (char *)param, tstrerror(c));
mnError("save %s failed, reason:%s", (char *)param, tstrerror(c));
} else {
int32_t rows = taos_affected_rows(result);
monitorDebug("save %s succ, rows:%d", (char *)param, rows);
mnDebug("save %s succ, rows:%d", (char *)param, rows);
}
taos_free_result(result);
......@@ -380,7 +380,7 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) {
pMon->totalConns, pMon->maxConns,
pMon->accessState);
monitorDebug("save account info, sql:%s", sql);
mnDebug("save account info, sql:%s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "account info");
}
......@@ -401,13 +401,13 @@ void monitorSaveLog(int32_t level, const char *const format, ...) {
len += sprintf(sql + len, "', '%s')", tsLocalEp);
sql[len++] = 0;
monitorDebug("save log, sql: %s", sql);
mnDebug("save log, sql: %s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "log");
}
void monitorExecuteSQL(char *sql) {
if (tsMonitor.state != MON_STATE_INITED) return;
monitorDebug("execute sql:%s", sql);
mnDebug("execute sql:%s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "sql");
}
......@@ -200,12 +200,6 @@ enum {
QUERY_RESULT_READY = 2,
};
typedef struct SMemRef {
int32_t ref;
void *mem;
void *imem;
} SMemRef;
typedef struct SQInfo {
void* signature;
int32_t code; // error code to returned to client
......
......@@ -1711,6 +1711,19 @@ _clean:
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
static void doFreeQueryHandle(SQInfo* pQInfo) {
SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
pRuntimeEnv->pQueryHandle = NULL;
pRuntimeEnv->pSecQueryHandle = NULL;
SMemRef* pMemRef = &pQInfo->memRef;
assert(pMemRef->ref == 0 && pMemRef->imem == NULL && pMemRef->mem == NULL);
}
static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
if (pRuntimeEnv->pQuery == NULL) {
return;
......@@ -1740,8 +1753,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo);
destroyResultBuf(pRuntimeEnv->pResultBuf);
tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
doFreeQueryHandle(pQInfo);
pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf);
taosTFree(pRuntimeEnv->keyBuf);
......@@ -3536,7 +3548,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
}
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
if (pRuntimeEnv->pSecQueryHandle == NULL) {
longjmp(pRuntimeEnv->env, terrno);
}
......@@ -3620,7 +3632,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
}
restoreTimeWindow(&pQInfo->tableGroupInfo, &cond);
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
if (pRuntimeEnv->pSecQueryHandle == NULL) {
longjmp(pRuntimeEnv->env, terrno);
}
......@@ -4462,7 +4474,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
terrno = TSDB_CODE_SUCCESS;
if (isFirstLastRowQuery(pQuery)) {
pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
// update the query time window
pQuery->window = cond.twindow;
......@@ -4484,9 +4496,9 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
}
}
} else if (isPointInterpoQuery(pQuery)) {
pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
} else {
pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
}
return terrno;
......@@ -4765,7 +4777,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
pRuntimeEnv->pQueryHandle = NULL;
}
pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
taosArrayDestroy(tx);
taosArrayDestroy(g1);
if (pRuntimeEnv->pQueryHandle == NULL) {
......@@ -4880,7 +4892,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
pRuntimeEnv->pQueryHandle = NULL;
}
pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo);
pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
taosArrayDestroy(tx);
taosArrayDestroy(g1);
......@@ -4946,7 +4958,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
// no need to update the lastkey for each table
pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
taosArrayDestroy(g1);
taosArrayDestroy(tx);
......@@ -5155,7 +5167,7 @@ static void doSaveContext(SQInfo *pQInfo) {
setupQueryRangeForReverseScan(pQInfo);
pRuntimeEnv->prevGroupId = INT32_MIN;
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
if (pRuntimeEnv->pSecQueryHandle == NULL) {
longjmp(pRuntimeEnv->env, terrno);
}
......@@ -6841,7 +6853,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex
int32_t code = TSDB_CODE_SUCCESS;
#if 0
#if _NON_BLOCKING_RETRIEVE
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
pthread_mutex_lock(&pQInfo->lock);
......@@ -6913,6 +6925,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
pQInfo->dataReady = QUERY_RESULT_NOT_READY;
if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
// here current thread hold the refcount, so it is safe to free tsdbQueryHandle.
doFreeQueryHandle(pQInfo);
*continueExec = false;
(*pRsp)->completed = 1; // notify no more result to client
} else {
......
......@@ -82,6 +82,7 @@ typedef struct {
int8_t oldInUse; // server EP inUse passed by app
int8_t redirect; // flag to indicate redirect
int8_t connType; // connection type
int64_t rid; // refId returned by taosAddRef
SRpcMsg *pRsp; // for synchronous API
tsem_t *pSem; // for synchronous API
SRpcEpSet *pSet; // for synchronous API
......@@ -220,8 +221,7 @@ static void rpcFree(void *p) {
free(p);
}
static void rpcInit(void) {
void rpcInit(void) {
tsProgressTimer = tsRpcTimer/2;
tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer;
tsRpcHeadSize = RPC_MSG_OVERHEAD;
......@@ -230,6 +230,11 @@ static void rpcInit(void) {
tsRpcRefId = taosOpenRef(200, rpcFree);
}
void rpcCleanup(void) {
taosCloseRef(tsRpcRefId);
tsRpcRefId = -1;
}
void *rpcOpen(const SRpcInit *pInit) {
SRpcInfo *pRpc;
......@@ -374,7 +379,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
}
void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
int64_t rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
SRpcInfo *pRpc = (SRpcInfo *)shandle;
SRpcReqContext *pContext;
......@@ -403,10 +408,11 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
// set the handle to pContext, so app can cancel the request
if (pMsg->handle) *((void **)pMsg->handle) = pContext;
taosAddRef(tsRpcRefId, pContext);
pContext->rid = taosAddRef(tsRpcRefId, pContext);
rpcSendReqToServer(pRpc, pContext);
return;
return pContext->rid;
}
void rpcSendResponse(const SRpcMsg *pRsp) {
......@@ -551,15 +557,14 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) {
return code;
}
void rpcCancelRequest(void *handle) {
SRpcReqContext *pContext = handle;
void rpcCancelRequest(int64_t rid) {
int code = taosAcquireRef(tsRpcRefId, pContext);
if (code < 0) return;
SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid);
if (pContext == NULL) return;
rpcCloseConn(pContext->pConn);
taosReleaseRef(tsRpcRefId, pContext);
taosReleaseRef(tsRpcRefId, rid);
}
static void rpcFreeMsg(void *msg) {
......@@ -628,7 +633,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
// if there is an outgoing message, free it
if (pConn->outType && pConn->pReqMsg) {
if (pConn->pContext) pConn->pContext->pConn = NULL;
taosRemoveRef(tsRpcRefId, pConn->pContext);
taosRemoveRef(tsRpcRefId, pConn->pContext->rid);
}
}
......@@ -1109,7 +1114,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
}
// free the request message
taosRemoveRef(tsRpcRefId, pContext);
taosRemoveRef(tsRpcRefId, pContext->rid);
}
static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) {
......@@ -1620,11 +1625,7 @@ static void rpcDecRef(SRpcInfo *pRpc)
tDebug("%s rpc resources are released", pRpc->label);
taosTFree(pRpc);
int count = atomic_sub_fetch_32(&tsRpcNum, 1);
if (count == 0) {
// taosCloseRef(tsRpcRefId);
// tsRpcInit = PTHREAD_ONCE_INIT; // windows compliling error
}
atomic_sub_fetch_32(&tsRpcNum, 1);
}
}
......@@ -141,6 +141,7 @@ typedef struct SSyncNode {
int8_t replica;
int8_t quorum;
uint32_t vgId;
int64_t rid;
void *ahandle;
int8_t selfIndex;
SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator
......
......@@ -142,14 +142,14 @@ void syncCleanUp() {
sInfo("sync module is cleaned up");
}
void *syncStart(const SSyncInfo *pInfo) {
int64_t syncStart(const SSyncInfo *pInfo) {
const SSyncCfg *pCfg = &pInfo->syncCfg;
SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1);
if (pNode == NULL) {
sError("no memory to allocate syncNode");
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
return -1;
}
tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
......@@ -170,10 +170,10 @@ void *syncStart(const SSyncInfo *pInfo) {
pNode->quorum = pCfg->quorum;
if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
int ret = taosAddRef(tsSyncRefId, pNode);
if (ret < 0) {
pNode->rid = taosAddRef(tsSyncRefId, pNode);
if (pNode->rid < 0) {
syncFreeNode(pNode);
return NULL;
return -1;
}
for (int i = 0; i < pCfg->replica; ++i) {
......@@ -187,8 +187,8 @@ void *syncStart(const SSyncInfo *pInfo) {
if (pNode->selfIndex < 0) {
sInfo("vgId:%d, this node is not configured", pNode->vgId);
terrno = TSDB_CODE_SYN_INVALID_CONFIG;
syncStop(pNode);
return NULL;
syncStop(pNode->rid);
return -1;
}
nodeVersion = pInfo->version; // set the initial version
......@@ -200,15 +200,15 @@ void *syncStart(const SSyncInfo *pInfo) {
if (pNode->pSyncFwds == NULL) {
sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId);
terrno = TAOS_SYSTEM_ERROR(errno);
syncStop(pNode);
return NULL;
syncStop(pNode->rid);
return -1;
}
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl);
if (pNode->pFwdTimer == NULL) {
sError("vgId:%d, failed to allocate timer", pNode->vgId);
syncStop(pNode);
return NULL;
syncStop(pNode->rid);
return -1;
}
syncAddArbitrator(pNode);
......@@ -218,15 +218,14 @@ void *syncStart(const SSyncInfo *pInfo) {
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
}
return pNode;
return pNode->rid;
}
void syncStop(void *param) {
SSyncNode *pNode = param;
void syncStop(int64_t rid) {
SSyncPeer *pPeer;
int ret = taosAcquireRef(tsSyncRefId, pNode);
if (ret < 0) return;
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return;
sInfo("vgId:%d, cleanup sync", pNode->vgId);
......@@ -245,16 +244,15 @@ void syncStop(void *param) {
pthread_mutex_unlock(&(pNode->mutex));
taosReleaseRef(tsSyncRefId, pNode);
taosRemoveRef(tsSyncRefId, pNode);
taosReleaseRef(tsSyncRefId, rid);
taosRemoveRef(tsSyncRefId, rid);
}
int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
SSyncNode *pNode = param;
int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
int i, j;
int ret = taosAcquireRef(tsSyncRefId, pNode);
if (ret < 0) return TSDB_CODE_SYN_INVALID_CONFIG;
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica,
pNode->replica);
......@@ -318,29 +316,25 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
syncRole[nodeRole]);
syncBroadcastStatus(pNode);
taosReleaseRef(tsSyncRefId, pNode);
taosReleaseRef(tsSyncRefId, rid);
return 0;
}
int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) {
SSyncNode *pNode = param;
int ret = taosAcquireRef(tsSyncRefId, pNode);
if (ret < 0) return 0;
int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int qtype) {
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return 0;
int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype);
taosReleaseRef(tsSyncRefId, pNode);
taosReleaseRef(tsSyncRefId, rid);
return code;
}
void syncConfirmForward(void *param, uint64_t version, int32_t code) {
SSyncNode *pNode = param;
int ret = taosAcquireRef(tsSyncRefId, pNode);
if (ret < 0) return;
void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return;
SSyncPeer *pPeer = pNode->pMaster;
if (pPeer && pNode->quorum > 1) {
......@@ -365,15 +359,14 @@ void syncConfirmForward(void *param, uint64_t version, int32_t code) {
}
}
taosReleaseRef(tsSyncRefId, pNode);
taosReleaseRef(tsSyncRefId, rid);
}
void syncRecover(void *param) {
SSyncNode *pNode = param;
void syncRecover(int64_t rid) {
SSyncPeer *pPeer;
int ret = taosAcquireRef(tsSyncRefId, pNode);
if (ret < 0) return;
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return;
// to do: add a few lines to check if recover is OK
// if take this node to unsync state, the whole system may not work
......@@ -393,14 +386,12 @@ void syncRecover(void *param) {
pthread_mutex_unlock(&(pNode->mutex));
taosReleaseRef(tsSyncRefId, pNode);
taosReleaseRef(tsSyncRefId, rid);
}
int syncGetNodesRole(void *param, SNodesRole *pNodesRole) {
SSyncNode *pNode = param;
int ret = taosAcquireRef(tsSyncRefId, pNode);
if (ret < 0) return -1;
int syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return -1;
pNodesRole->selfIndex = pNode->selfIndex;
for (int i = 0; i < pNode->replica; ++i) {
......@@ -408,7 +399,7 @@ int syncGetNodesRole(void *param, SNodesRole *pNodesRole) {
pNodesRole->role[i] = pNode->peerInfo[i]->role;
}
taosReleaseRef(tsSyncRefId, pNode);
taosReleaseRef(tsSyncRefId, rid);
return 0;
}
......@@ -455,7 +446,7 @@ void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1);
int syncDecPeerRef(SSyncPeer *pPeer) {
if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
taosReleaseRef(tsSyncRefId, pPeer->pSyncNode);
taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
sDebug("%s, resource is freed", pPeer->id);
taosTFree(pPeer->watchFd);
......@@ -512,7 +503,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer);
}
taosAcquireRef(tsSyncRefId, pNode);
taosAcquireRef(tsSyncRefId, pNode->rid);
return pPeer;
}
......@@ -1105,7 +1096,7 @@ static void syncProcessBrokenLink(void *param) {
SSyncPeer *pPeer = param;
SSyncNode *pNode = pPeer->pSyncNode;
if (taosAcquireRef(tsSyncRefId, pNode) < 0) return;
if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return;
pthread_mutex_lock(&(pNode->mutex));
sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
......@@ -1116,7 +1107,7 @@ static void syncProcessBrokenLink(void *param) {
}
pthread_mutex_unlock(&(pNode->mutex));
taosReleaseRef(tsSyncRefId, pNode);
taosReleaseRef(tsSyncRefId, pNode->rid);
}
static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
......@@ -1184,10 +1175,9 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code
}
static void syncMonitorFwdInfos(void *param, void *tmrId) {
SSyncNode *pNode = param;
int ret = taosAcquireRef(tsSyncRefId, pNode);
if ( ret < 0) return;
int64_t rid = (int64_t) param;
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return;
SSyncFwds *pSyncFwds = pNode->pSyncFwds;
......@@ -1206,10 +1196,10 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
pthread_mutex_unlock(&(pNode->mutex));
}
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl);
}
taosReleaseRef(tsSyncRefId, pNode);
taosReleaseRef(tsSyncRefId, rid);
}
static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtype) {
......
......@@ -30,7 +30,7 @@ int dataFd = -1;
void * qhandle = NULL;
int walNum = 0;
uint64_t tversion = 0;
void * syncHandle;
int64_t syncHandle;
int role;
int nodeId;
char path[256];
......
......@@ -198,6 +198,8 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem)
if (pIMem != NULL) {
tsdbUnRefMemTable(pRepo, pIMem);
}
tsdbDebug("vgId:%d utake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem);
}
void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
......
......@@ -20,7 +20,6 @@
#include "exception.h"
#include "../../query/inc/qAst.h" // todo move to common module
#include "../../query/inc/qExecutor.h" // todo move to common module
#include "tlosertree.h"
#include "tsdb.h"
#include "tsdbMain.h"
......@@ -120,8 +119,9 @@ typedef struct STsdbQueryHandle {
SDataCols *pDataCols; // in order to hold current file data block
int32_t allocSize; // allocated data block size
SMemTable *mem; // mem-table
SMemTable *imem; // imem-table, acquired from snapshot
SMemRef *pMemRef;
// SMemTable *mem; // mem-table
// SMemTable *imem; // imem-table, acquired from snapshot
SArray *defaultLoadColumn;// default load column
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
......@@ -184,26 +184,26 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS
return pLocalIdList;
}
static void tsdbMayTakeMemSnapshot(TsdbQueryHandleT pHandle) {
STsdbQueryHandle* pSecQueryHandle = (STsdbQueryHandle*) pHandle;
SQInfo *pQInfo = (SQInfo *)(pSecQueryHandle->qinfo);
static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL);
if (pQInfo->memRef.ref++ == 0) {
tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem);
pQInfo->memRef.mem = pSecQueryHandle->mem;
pQInfo->memRef.imem = pSecQueryHandle->imem;
} else {
pSecQueryHandle->mem = (SMemTable *)(pQInfo->memRef.mem);
pSecQueryHandle->imem = (SMemTable *)(pQInfo->memRef.imem);
SMemRef* pMemRef = pQueryHandle->pMemRef;
if (pQueryHandle->pMemRef->ref++ == 0) {
tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem));
}
}
static void tsdbMayUnTakeMemSnapshot(TsdbQueryHandleT pHandle) {
STsdbQueryHandle* pSecQueryHandle = (STsdbQueryHandle*) pHandle;
SQInfo *pQInfo = (SQInfo *)(pSecQueryHandle->qinfo);
if (--pQInfo->memRef.ref == 0) {
tsdbUnTakeMemSnapShot(pSecQueryHandle->pTsdb, pSecQueryHandle->mem, pSecQueryHandle->imem);
static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL);
SMemRef* pMemRef = pQueryHandle->pMemRef;
if (--pMemRef->ref == 0) {
tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pMemRef->mem, pMemRef->imem);
pMemRef->mem = NULL;
pMemRef->imem = NULL;
}
pQueryHandle->pMemRef = NULL;
}
static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) {
size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList);
......@@ -270,7 +270,7 @@ static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey)
return pNew;
}
static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo) {
static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) {
STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
if (pQueryHandle == NULL) {
goto out_of_memory;
......@@ -288,13 +288,14 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock;
pQueryHandle->allocSize = 0;
pQueryHandle->locateStart = false;
pQueryHandle->pMemRef = pMemRef;
if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) {
goto out_of_memory;
}
tsdbMayTakeMemSnapshot(pQueryHandle);
assert(pCond != NULL && pCond->numOfCols > 0);
assert(pCond != NULL && pCond->numOfCols > 0 && pMemRef != NULL);
if (ASCENDING_TRAVERSE(pCond->order)) {
assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey);
......@@ -348,8 +349,8 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
return NULL;
}
TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) {
STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo);
TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) {
STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo, pRef);
STsdbMeta* pMeta = tsdbGetMeta(tsdb);
assert(pMeta != NULL);
......@@ -366,7 +367,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
return (TsdbQueryHandleT) pQueryHandle;
}
TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) {
TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) {
pCond->twindow = changeTableGroupByLastrow(groupList);
// no qualified table
......@@ -374,7 +375,7 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
return NULL;
}
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pMemRef);
assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey);
return pQueryHandle;
......@@ -396,8 +397,8 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
return res;
}
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) {
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) {
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef);
if (pQueryHandle != NULL) {
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
changeQueryHandleForInterpQuery(pQueryHandle);
......@@ -417,7 +418,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
int32_t order = pHandle->order;
// no data in buffer, abort
if (pHandle->mem == NULL && pHandle->imem == NULL) {
if (pHandle->pMemRef->mem == NULL && pHandle->pMemRef->imem == NULL) {
return false;
}
......@@ -426,16 +427,19 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
STableData* pMem = NULL;
STableData* pIMem = NULL;
if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables) {
pMem = pHandle->mem->tData[pCheckInfo->tableId.tid];
SMemTable* pMemT = pHandle->pMemRef->mem;
SMemTable* pIMemT = pHandle->pMemRef->imem;
if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) {
pMem = pMemT->tData[pCheckInfo->tableId.tid];
if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid
pCheckInfo->iter =
tSkipListCreateIterFromVal(pMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
}
}
if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables) {
pIMem = pHandle->imem->tData[pCheckInfo->tableId.tid];
if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) {
pIMem = pIMemT->tData[pCheckInfo->tableId.tid];
if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid
pCheckInfo->iiter =
tSkipListCreateIterFromVal(pIMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
......@@ -2029,7 +2033,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
}
STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo);
STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef);
taosTFree(cond.colList);
......
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TFILE_H
#define TDENGINE_TFILE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <unistd.h>
// init taos file module
int32_t tfinit();
// clean up taos file module
void tfcleanup();
// the same syntax as UNIX standard open/close/read/write
// but FD is int64_t and will never be reused
int64_t tfopen(const char *pathname, int32_t flags);
int64_t tfclose(int64_t tfd);
int64_t tfwrite(int64_t tfd, void *buf, int64_t count);
int64_t tfread(int64_t tfd, void *buf, int64_t count);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TREF_H
......@@ -21,38 +21,48 @@
extern "C" {
#endif
// open an instance, return refId which will be used by other APIs
int taosOpenRef(int max, void (*fp)(void *));
// open a reference set, max is the mod used by hash, fp is the pointer to free resource function
// return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately
int taosOpenRef(int max, void (*fp)(void *));
// close the Ref instance
void taosCloseRef(int refId);
// close the reference set, refId is the return value by taosOpenRef
// return 0 if success. On error, -1 is returned, and terrno is set appropriately
int taosCloseRef(int refId);
// add ref, p is the pointer to resource or pointer ID
int taosAddRef(int refId, void *p);
#define taosRemoveRef taosReleaseRef
// return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately
int64_t taosAddRef(int refId, void *p);
// acquire ref, p is the pointer to resource or pointer ID
int taosAcquireRef(int refId, void *p);
// remove ref, rid is the reference ID returned by taosAddRef
// return 0 if success. On error, -1 is returned, and terrno is set appropriately
int taosRemoveRef(int rsetId, int64_t rid);
// release ref, p is the pointer to resource or pinter ID
void taosReleaseRef(int refId, void *p);
// acquire ref, rid is the reference ID returned by taosAddRef
// return the resource p. On error, NULL is returned, and terrno is set appropriately
void *taosAcquireRef(int rsetId, int64_t rid);
// return the first if p is null, otherwise return the next after p
void *taosIterateRef(int refId, void *p);
// release ref, rid is the reference ID returned by taosAddRef
// return 0 if success. On error, -1 is returned, and terrno is set appropriately
int taosReleaseRef(int rsetId, int64_t rid);
// return the first reference if rid is 0, otherwise return the next after current reference.
// if return value is NULL, it means list is over(if terrno is set, it means error happens)
void *taosIterateRef(int rsetId, int64_t rid);
// return the number of references in system
int taosListRef();
/* sample code to iterate the refs
void demoIterateRefs(int refId) {
void demoIterateRefs(int rsetId) {
void *p = taosIterateRef(refId, NULL);
void *p = taosIterateRef(refId, 0);
while (p) {
// process P
// get the rid from p
p = taosIterateRef(refId, p);
p = taosIterateRef(rsetId, rid);
}
}
......
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "taoserror.h"
#include "tulog.h"
#include "tutil.h"
#include "tref.h"
static int32_t tsFileRsetId = -1;
static void taosCloseFile(void *p) {
close((int32_t)(uintptr_t)p);
}
int32_t tfinit() {
tsFileRsetId = taosOpenRef(2000, taosCloseFile);
return tsFileRsetId;
}
void tfcleanup() {
if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
tsFileRsetId = -1;
}
int64_t tfopen(const char *pathname, int32_t flags) {
int32_t fd = open(pathname, flags);
if (fd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
void *p = (void *)(int64_t)fd;
int64_t rid = taosAddRef(tsFileRsetId, p);
if (rid < 0) close(fd);
return rid;
}
int64_t tfclose(int64_t tfd) {
return taosRemoveRef(tsFileRsetId, tfd);
}
int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int64_t ret = taosWrite(fd, buf, count);
if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
int64_t tfread(int64_t tfd, void *buf, int64_t count) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int64_t ret = taosRead(fd, buf, count);
if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
......@@ -24,19 +24,22 @@
#define TSDB_REF_STATE_DELETED 2
typedef struct SRefNode {
struct SRefNode *prev;
struct SRefNode *next;
void *p;
int32_t count;
struct SRefNode *prev; // previous node
struct SRefNode *next; // next node
void *p; // pointer to resource protected,
int64_t rid; // reference ID
int32_t count; // number of references
int removed; // 1: removed
} SRefNode;
typedef struct {
SRefNode **nodeList;
int state; // 0: empty, 1: active; 2: deleted
int refId;
int max;
int32_t count; // total number of SRefNodes in this set
int64_t *lockedBy;
SRefNode **nodeList; // array of SRefNode linked list
int state; // 0: empty, 1: active; 2: deleted
int rsetId; // refSet ID, global unique
int64_t rid; // increase by one for each new reference
int max; // mod
int32_t count; // total number of SRefNodes in this set
int64_t *lockedBy;
void (*fp)(void *);
} SRefSet;
......@@ -47,54 +50,58 @@ static int tsRefSetNum = 0;
static int tsNextId = 0;
static void taosInitRefModule(void);
static int taosHashRef(SRefSet *pSet, void *p);
static void taosLockList(int64_t *lockedBy);
static void taosUnlockList(int64_t *lockedBy);
static void taosIncRefCount(SRefSet *pSet);
static void taosDecRefCount(SRefSet *pSet);
static void taosIncRsetCount(SRefSet *pSet);
static void taosDecRsetCount(SRefSet *pSet);
static int taosDecRefCount(int rsetId, int64_t rid, int remove);
int taosOpenRef(int max, void (*fp)(void *))
{
SRefNode **nodeList;
SRefSet *pSet;
int64_t *lockedBy;
int i, refId;
int i, rsetId;
pthread_once(&tsRefModuleInit, taosInitRefModule);
nodeList = calloc(sizeof(SRefNode *), (size_t)max);
if (nodeList == NULL) {
return TSDB_CODE_REF_NO_MEMORY;
if (nodeList == NULL) {
terrno = TSDB_CODE_REF_NO_MEMORY;
return -1;
}
lockedBy = calloc(sizeof(int64_t), (size_t)max);
if (lockedBy == NULL) {
free(nodeList);
return TSDB_CODE_REF_NO_MEMORY;
terrno = TSDB_CODE_REF_NO_MEMORY;
return -1;
}
pthread_mutex_lock(&tsRefMutex);
for (i = 0; i < TSDB_REF_OBJECTS; ++i) {
tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS;
if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId
if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break;
}
if (i < TSDB_REF_OBJECTS) {
refId = tsNextId;
pSet = tsRefSetList + refId;
taosIncRefCount(pSet);
rsetId = tsNextId;
pSet = tsRefSetList + rsetId;
pSet->max = max;
pSet->nodeList = nodeList;
pSet->lockedBy = lockedBy;
pSet->fp = fp;
pSet->rid = 1;
pSet->rsetId = rsetId;
pSet->state = TSDB_REF_STATE_ACTIVE;
pSet->refId = refId;
taosIncRsetCount(pSet);
tsRefSetNum++;
uTrace("refId:%d is opened, max:%d, fp:%p refSetNum:%d", refId, max, fp, tsRefSetNum);
uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum);
} else {
refId = TSDB_CODE_REF_FULL;
rsetId = TSDB_CODE_REF_FULL;
free (nodeList);
free (lockedBy);
uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum);
......@@ -102,121 +109,128 @@ int taosOpenRef(int max, void (*fp)(void *))
pthread_mutex_unlock(&tsRefMutex);
return refId;
return rsetId;
}
void taosCloseRef(int refId)
int taosCloseRef(int rsetId)
{
SRefSet *pSet;
int deleted = 0;
if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
uTrace("refId:%d is invalid, out of range", refId);
return;
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d is invalid, out of range", rsetId);
terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
}
pSet = tsRefSetList + refId;
pSet = tsRefSetList + rsetId;
pthread_mutex_lock(&tsRefMutex);
if (pSet->state == TSDB_REF_STATE_ACTIVE) {
pSet->state = TSDB_REF_STATE_DELETED;
deleted = 1;
uTrace("refId:%d is closed, count:%d", refId, pSet->count);
uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count);
} else {
uTrace("refId:%d is already closed, count:%d", refId, pSet->count);
uTrace("rsetId:%d is already closed, count:%d", rsetId, pSet->count);
}
pthread_mutex_unlock(&tsRefMutex);
if (deleted) taosDecRefCount(pSet);
if (deleted) taosDecRsetCount(pSet);
return 0;
}
int taosAddRef(int refId, void *p)
int64_t taosAddRef(int rsetId, void *p)
{
int hash;
SRefNode *pNode;
SRefSet *pSet;
int64_t rid = 0;
if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
uTrace("refId:%d p:%p failed to add, refId not valid", refId, p);
return TSDB_CODE_REF_INVALID_ID;
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d p:%p failed to add, rsetId not valid", rsetId, p);
terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
}
pSet = tsRefSetList + refId;
taosIncRefCount(pSet);
pSet = tsRefSetList + rsetId;
taosIncRsetCount(pSet);
if (pSet->state != TSDB_REF_STATE_ACTIVE) {
taosDecRefCount(pSet);
uTrace("refId:%d p:%p failed to add, not active", refId, p);
return TSDB_CODE_REF_ID_REMOVED;
taosDecRsetCount(pSet);
uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p);
terrno = TSDB_CODE_REF_ID_REMOVED;
return -1;
}
int code = 0;
hash = taosHashRef(pSet, p);
pNode = calloc(sizeof(SRefNode), 1);
if (pNode == NULL) {
terrno = TSDB_CODE_REF_NO_MEMORY;
return -1;
}
rid = atomic_add_fetch_64(&pSet->rid, 1);
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->p == p)
break;
pNode->p = p;
pNode->rid = rid;
pNode->count = 1;
pNode = pNode->next;
}
if (pNode) {
code = TSDB_CODE_REF_ALREADY_EXIST;
uTrace("refId:%d p:%p is already there, faild to add", refId, p);
} else {
pNode = calloc(sizeof(SRefNode), 1);
if (pNode) {
pNode->p = p;
pNode->count = 1;
pNode->prev = 0;
pNode->next = pSet->nodeList[hash];
if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode;
pSet->nodeList[hash] = pNode;
uTrace("refId:%d p:%p is added, count:%d malloc mem: %p", refId, p, pSet->count, pNode);
} else {
code = TSDB_CODE_REF_NO_MEMORY;
uTrace("refId:%d p:%p is not added, since no memory", refId, p);
}
}
pNode->prev = NULL;
pNode->next = pSet->nodeList[hash];
if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode;
pSet->nodeList[hash] = pNode;
if (code < 0) taosDecRefCount(pSet);
uTrace("rsetId:%d p:%p rid:%" PRId64 " is added, count:%d", rsetId, p, rid, pSet->count);
taosUnlockList(pSet->lockedBy+hash);
return code;
return rid;
}
int taosRemoveRef(int rsetId, int64_t rid)
{
return taosDecRefCount(rsetId, rid, 1);
}
int taosAcquireRef(int refId, void *p)
// if rid is 0, return the first p in hash list, otherwise, return the next after current rid
void *taosAcquireRef(int rsetId, int64_t rid)
{
int hash, code = 0;
int hash;
SRefNode *pNode;
SRefSet *pSet;
void *p = NULL;
if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
uTrace("refId:%d p:%p failed to acquire, refId not valid", refId, p);
return TSDB_CODE_REF_INVALID_ID;
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid);
terrno = TSDB_CODE_REF_INVALID_ID;
return NULL;
}
pSet = tsRefSetList + refId;
taosIncRefCount(pSet);
if (rid <= 0) {
uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rid not valid", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
return NULL;
}
pSet = tsRefSetList + rsetId;
taosIncRsetCount(pSet);
if (pSet->state != TSDB_REF_STATE_ACTIVE) {
uTrace("refId:%d p:%p failed to acquire, not active", refId, p);
taosDecRefCount(pSet);
return TSDB_CODE_REF_ID_REMOVED;
uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, not active", rsetId, rid);
taosDecRsetCount(pSet);
terrno = TSDB_CODE_REF_ID_REMOVED;
return NULL;
}
hash = taosHashRef(pSet, p);
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->p == p) {
if (pNode->rid == rid) {
break;
}
......@@ -224,117 +238,76 @@ int taosAcquireRef(int refId, void *p)
}
if (pNode) {
pNode->count++;
uTrace("refId:%d p:%p is acquired", refId, p);
if (pNode->removed == 0) {
pNode->count++;
p = pNode->p;
uTrace("rsetId:%d p:%p rid:%" PRId64 " is acquired", rsetId, pNode->p, rid);
} else {
terrno = TSDB_CODE_REF_NOT_EXIST;
uTrace("rsetId:%d p:%p rid:%" PRId64 " is already removed, failed to acquire", rsetId, pNode->p, rid);
}
} else {
code = TSDB_CODE_REF_NOT_EXIST;
uTrace("refId:%d p:%p is not there, failed to acquire", refId, p);
terrno = TSDB_CODE_REF_NOT_EXIST;
uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to acquire", rsetId, rid);
}
taosUnlockList(pSet->lockedBy+hash);
taosDecRefCount(pSet);
taosDecRsetCount(pSet);
return code;
return p;
}
void taosReleaseRef(int refId, void *p)
int taosReleaseRef(int rsetId, int64_t rid)
{
int hash;
SRefNode *pNode;
SRefSet *pSet;
int released = 0;
if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
uTrace("refId:%d p:%p failed to release, refId not valid", refId, p);
return;
}
pSet = tsRefSetList + refId;
if (pSet->state == TSDB_REF_STATE_EMPTY) {
uTrace("refId:%d p:%p failed to release, cleaned", refId, p);
return;
}
hash = taosHashRef(pSet, p);
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->p == p)
break;
pNode = pNode->next;
}
if (pNode) {
pNode->count--;
if (pNode->count == 0) {
if ( pNode->prev ) {
pNode->prev->next = pNode->next;
} else {
pSet->nodeList[hash] = pNode->next;
}
if ( pNode->next ) {
pNode->next->prev = pNode->prev;
}
(*pSet->fp)(pNode->p);
free(pNode);
released = 1;
uTrace("refId:%d p:%p is removed, count:%d, free mem: %p", refId, p, pSet->count, pNode);
} else {
uTrace("refId:%d p:%p is released", refId, p);
}
} else {
uTrace("refId:%d p:%p is not there, failed to release", refId, p);
}
taosUnlockList(pSet->lockedBy+hash);
if (released) taosDecRefCount(pSet);
return taosDecRefCount(rsetId, rid, 0);
}
// if p is NULL, return the first p in hash list, otherwise, return the next after p
void *taosIterateRef(int refId, void *p) {
// if rid is 0, return the first p in hash list, otherwise, return the next after current rid
void *taosIterateRef(int rsetId, int64_t rid) {
SRefNode *pNode = NULL;
SRefSet *pSet;
if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
uTrace("refId:%d p:%p failed to iterate, refId not valid", refId, p);
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rsetId not valid", rsetId, rid);
terrno = TSDB_CODE_REF_INVALID_ID;
return NULL;
}
if (rid <= 0) {
uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rid not valid", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
return NULL;
}
pSet = tsRefSetList + refId;
taosIncRefCount(pSet);
pSet = tsRefSetList + rsetId;
taosIncRsetCount(pSet);
if (pSet->state != TSDB_REF_STATE_ACTIVE) {
uTrace("refId:%d p:%p failed to iterate, not active", refId, p);
taosDecRefCount(pSet);
uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rset not active", rsetId, rid);
terrno = TSDB_CODE_REF_ID_REMOVED;
taosDecRsetCount(pSet);
return NULL;
}
int hash = 0;
if (p) {
hash = taosHashRef(pSet, p);
if (rid > 0) {
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->p == p) break;
if (pNode->rid == rid) break;
pNode = pNode->next;
}
if (pNode == NULL) {
uError("refId:%d p:%p not there, quit", refId, p);
uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
taosUnlockList(pSet->lockedBy+hash);
return NULL;
}
// p is there
// rid is there
pNode = pNode->next;
if (pNode == NULL) {
taosUnlockList(pSet->lockedBy+hash);
......@@ -356,14 +329,14 @@ void *taosIterateRef(int refId, void *p) {
pNode->count++; // acquire it
newP = pNode->p;
taosUnlockList(pSet->lockedBy+hash);
uTrace("refId:%d p:%p is returned", refId, p);
uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid);
} else {
uTrace("refId:%d p:%p the list is over", refId, p);
uTrace("rsetId:%d the list is over", rsetId);
}
if (p) taosReleaseRef(refId, p); // release the current one
if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one
taosDecRefCount(pSet);
taosDecRsetCount(pSet);
return newP;
}
......@@ -381,13 +354,13 @@ int taosListRef() {
if (pSet->state == TSDB_REF_STATE_EMPTY)
continue;
uInfo("refId:%d state:%d count::%d", i, pSet->state, pSet->count);
uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count);
for (int j=0; j < pSet->max; ++j) {
pNode = pSet->nodeList[j];
while (pNode) {
uInfo("refId:%d p:%p count:%d", i, pNode->p, pNode->count);
uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count);
pNode = pNode->next;
num++;
}
......@@ -399,21 +372,78 @@ int taosListRef() {
return num;
}
static int taosHashRef(SRefSet *pSet, void *p)
{
int hash = 0;
int64_t v = (int64_t)p;
static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
int hash;
SRefSet *pSet;
SRefNode *pNode;
int released = 0;
int code = 0;
if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid);
terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
}
if (rid <= 0) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rid not valid", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
return -1;
}
pSet = tsRefSetList + rsetId;
if (pSet->state == TSDB_REF_STATE_EMPTY) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, cleaned", rsetId, rid);
terrno = TSDB_CODE_REF_ID_REMOVED;
return -1;
}
for (int i = 0; i < sizeof(v); ++i) {
hash += (int)(v & 0xFFFF);
v = v >> 16;
i = i + 2;
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->rid == rid)
break;
pNode = pNode->next;
}
hash = hash % pSet->max;
if (pNode) {
pNode->count--;
if (remove) pNode->removed = 1;
return hash;
}
if (pNode->count <= 0) {
if (pNode->prev) {
pNode->prev->next = pNode->next;
} else {
pSet->nodeList[hash] = pNode->next;
}
if (pNode->next) {
pNode->next->prev = pNode->prev;
}
(*pSet->fp)(pNode->p);
uTrace("rsetId:%d p:%p rid:%" PRId64 "is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
free(pNode);
released = 1;
} else {
uTrace("rsetId:%d p:%p rid:%" PRId64 "is released, count:%d", rsetId, pNode->p, rid, pNode->count);
}
} else {
uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
code = -1;
}
taosUnlockList(pSet->lockedBy+hash);
if (released) taosDecRsetCount(pSet);
return code;
}
static void taosLockList(int64_t *lockedBy) {
int64_t tid = taosGetPthreadId();
......@@ -436,14 +466,14 @@ static void taosInitRefModule(void) {
pthread_mutex_init(&tsRefMutex, NULL);
}
static void taosIncRefCount(SRefSet *pSet) {
static void taosIncRsetCount(SRefSet *pSet) {
atomic_add_fetch_32(&pSet->count, 1);
uTrace("refId:%d inc count:%d", pSet->refId, pSet->count);
// uTrace("rsetId:%d inc count:%d", pSet->rsetId, count);
}
static void taosDecRefCount(SRefSet *pSet) {
static void taosDecRsetCount(SRefSet *pSet) {
int32_t count = atomic_sub_fetch_32(&pSet->count, 1);
uTrace("refId:%d dec count:%d", pSet->refId, pSet->count);
// uTrace("rsetId:%d dec count:%d", pSet->rsetId, count);
if (count > 0) return;
......@@ -458,7 +488,7 @@ static void taosDecRefCount(SRefSet *pSet) {
taosTFree(pSet->lockedBy);
tsRefSetNum--;
uTrace("refId:%d is cleaned, refSetNum:%d count:%d", pSet->refId, tsRefSetNum, pSet->count);
uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count);
}
pthread_mutex_unlock(&tsRefMutex);
......
......@@ -11,106 +11,119 @@
#include "tulog.h"
typedef struct {
int refNum;
int steps;
int refId;
void **p;
int refNum;
int steps;
int rsetId;
int64_t rid;
void **p;
} SRefSpace;
void iterateRefs(int refId) {
void iterateRefs(int rsetId) {
int count = 0;
void *p = taosIterateRef(refId, NULL);
void *p = taosIterateRef(rsetId, NULL);
while (p) {
// process P
count++;
p = taosIterateRef(refId, p);
p = taosIterateRef(rsetId, p);
}
printf(" %d ", count);
}
void *takeRefActions(void *param) {
void *addRef(void *param) {
SRefSpace *pSpace = (SRefSpace *)param;
int code, id;
int id;
int64_t rid;
for (int i=0; i < pSpace->steps; ++i) {
printf("s");
printf("a");
id = random() % pSpace->refNum;
code = taosAddRef(pSpace->refId, pSpace->p[id]);
usleep(1);
id = random() % pSpace->refNum;
code = taosAcquireRef(pSpace->refId, pSpace->p[id]);
if (code >= 0) {
usleep(id % 5 + 1);
taosReleaseRef(pSpace->refId, pSpace->p[id]);
if (pSpace->rid[id] <= 0) {
pSpace->p[id] = malloc(128);
pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]);
}
usleep(100);
}
return NULL;
}
void *removeRef(void *param) {
SRefSpace *pSpace = (SRefSpace *)param;
int id;
int64_t rid;
for (int i=0; i < pSpace->steps; ++i) {
printf("d");
id = random() % pSpace->refNum;
taosRemoveRef(pSpace->refId, pSpace->p[id]);
usleep(id %5 + 1);
if (pSpace->rid[id] > 0) {
code = taosRemoveRef(pSpace->rsetId, pSpace->rid[id]);
if (code == 0) pSpace->rid[id] = 0;
}
usleep(100);
}
return NULL;
}
void *acquireRelease(void *param) {
SRefSpace *pSpace = (SRefSpace *)param;
int id;
int64_t rid;
for (int i=0; i < pSpace->steps; ++i) {
printf("a");
id = random() % pSpace->refNum;
code = taosAcquireRef(pSpace->refId, pSpace->p[id]);
code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]);
if (code >= 0) {
usleep(id % 5 + 1);
taosReleaseRef(pSpace->refId, pSpace->p[id]);
taosReleaseRef(pSpace->rsetId, pSpace->p[id]);
}
id = random() % pSpace->refNum;
iterateRefs(id);
}
for (int i=0; i < pSpace->refNum; ++i) {
taosRemoveRef(pSpace->refId, pSpace->p[i]);
}
//uInfo("refId:%d thread exits", pSpace->refId);
return NULL;
}
void myfree(void *p) {
return;
free(p);
}
void *openRefSpace(void *param) {
SRefSpace *pSpace = (SRefSpace *)param;
printf("c");
pSpace->refId = taosOpenRef(50, myfree);
pSpace->rsetId = taosOpenRef(50, myfree);
if (pSpace->refId < 0) {
printf("failed to open ref, reson:%s\n", tstrerror(pSpace->refId));
if (pSpace->rsetId < 0) {
printf("failed to open ref, reson:%s\n", tstrerror(pSpace->rsetId));
return NULL;
}
pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum);
for (int i=0; i<pSpace->refNum; ++i) {
pSpace->p[i] = (void *) malloc(128);
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pthread_t thread1, thread2, thread3;
pthread_create(&(thread1), &thattr, takeRefActions, (void *)(pSpace));
pthread_create(&(thread2), &thattr, takeRefActions, (void *)(pSpace));
pthread_create(&(thread3), &thattr, takeRefActions, (void *)(pSpace));
pthread_create(&(thread1), &thattr, addRef, (void *)(pSpace));
pthread_create(&(thread2), &thattr, removeRef, (void *)(pSpace));
pthread_create(&(thread3), &thattr, acquireRelease, (void *)(pSpace));
pthread_join(thread1, NULL);
pthread_join(thread2, NULL);
pthread_join(thread3, NULL);
taosCloseRef(pSpace->refId);
for (int i=0; i<pSpace->refNum; ++i) {
free(pSpace->p[i]);
taosRemoveRef(pSpace->rsetId, pSpace->rid[i]);
}
uInfo("refId:%d main thread exit", pSpace->refId);
taosCloseRef(pSpace->rsetId);
uInfo("rsetId:%d main thread exit", pSpace->rsetId);
free(pSpace->p);
pSpace->p = NULL;
......@@ -140,7 +153,7 @@ int main(int argc, char *argv[]) {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-n]: number of references, default: %d\n", refNum);
printf(" [-s]: steps to run for each reference, default: %d\n", steps);
printf(" [-t]: number of refIds running in parallel, default: %d\n", threads);
printf(" [-t]: number of rsetIds running in parallel, default: %d\n", threads);
printf(" [-l]: number of loops, default: %d\n", loops);
printf(" [-d]: debugFlag, default: %d\n", uDebugFlag);
exit(0);
......
......@@ -41,13 +41,13 @@ typedef struct {
int8_t status;
int8_t role;
int8_t accessState;
int64_t version; // current version
int64_t fversion; // version on saved data file
uint64_t version; // current version
uint64_t fversion; // version on saved data file
void *wqueue;
void *rqueue;
void *wal;
void *tsdb;
void *sync;
int64_t sync;
void *events;
void *cq; // continuous query
int32_t cfgVersion;
......
......@@ -22,7 +22,6 @@
#include "tsdb.h"
#include "dnode.h"
#include "vnodeInt.h"
#include "vnodeVersion.h"
#include "vnodeCfg.h"
static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
......@@ -40,8 +39,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression;
pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel;
pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod;
pVnode->walCfg.wals = vnodeMsg->cfg.wals;
pVnode->walCfg.keep = 0;
pVnode->walCfg.keep = TAOS_WAL_NOT_KEEP;
pVnode->syncCfg.replica = vnodeMsg->cfg.replications;
pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum;
......
......@@ -15,17 +15,11 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "tcache.h"
#include "cJSON.h"
#include "dnode.h"
#include "hash.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tglobal.h"
#include "trpc.h"
#include "tsdb.h"
#include "ttimer.h"
#include "tutil.h"
#include "vnode.h"
#include "vnodeInt.h"
......@@ -34,7 +28,7 @@
#include "vnodeCfg.h"
#include "vnodeVersion.h"
static SHashObj*tsDnodeVnodesHash;
static SHashObj*tsVnodesHash;
static void vnodeCleanUp(SVnodeObj *pVnode);
static int vnodeProcessTsdbStatus(void *arg, int status);
static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion);
......@@ -44,12 +38,12 @@ static void vnodeCtrlFlow(void *handle, int32_t mseconds);
static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion);
#ifndef _SYNC
tsync_h syncStart(const SSyncInfo *info) { return NULL; }
int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; }
void syncStop(tsync_h shandle) {}
int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
int syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
int64_t syncStart(const SSyncInfo *info) { return NULL; }
int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype) { return 0; }
void syncStop(int64_t rid) {}
int32_t syncReconfig(int64_t rid, const SSyncCfg * cfg) { return 0; }
int syncGetNodesRole(int64_t rid, SNodesRole * cfg) { return 0; }
void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {}
#endif
char* vnodeStatus[] = {
......@@ -67,8 +61,8 @@ int32_t vnodeInitResources() {
vnodeInitWriteFp();
vnodeInitReadFp();
tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
if (tsDnodeVnodesHash == NULL) {
tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
if (tsVnodesHash == NULL) {
vError("failed to init vnode list");
return TSDB_CODE_VND_OUT_OF_MEMORY;
}
......@@ -77,10 +71,10 @@ int32_t vnodeInitResources() {
}
void vnodeCleanupResources() {
if (tsDnodeVnodesHash != NULL) {
if (tsVnodesHash != NULL) {
vDebug("vnode list is cleanup");
taosHashCleanup(tsDnodeVnodesHash);
tsDnodeVnodesHash = NULL;
taosHashCleanup(tsVnodesHash);
tsVnodesHash = NULL;
}
syncCleanUp();
......@@ -233,6 +227,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
pVnode->vgId = vnode;
pVnode->status = TAOS_VN_STATUS_INIT;
pVnode->fversion = 0;
pVnode->version = 0;
pVnode->tsdbCfg.tsdbId = pVnode->vgId;
pVnode->rootDir = strdup(rootDir);
......@@ -294,6 +289,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
vnodeCleanUp(pVnode);
return terrno;
} else {
pVnode->fversion = 0;
pVnode->version = 0;
}
}
......@@ -308,6 +304,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
walRestore(pVnode->wal, pVnode, vnodeProcessWrite);
if (pVnode->version == 0) {
pVnode->fversion = 0;
pVnode->version = walGetVersion(pVnode->wal);
}
......@@ -331,7 +328,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
#ifndef _SYNC
pVnode->role = TAOS_SYNC_ROLE_MASTER;
#else
if (pVnode->sync == NULL) {
if (pVnode->sync <= 0) {
vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
tstrerror(terrno));
vnodeCleanUp(pVnode);
......@@ -349,7 +346,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
pVnode->status = TAOS_VN_STATUS_READY;
vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode);
taosHashPut(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
return TSDB_CODE_SUCCESS;
}
......@@ -441,7 +438,7 @@ void vnodeRelease(void *pVnodeRaw) {
tsem_destroy(&pVnode->sem);
free(pVnode);
int32_t count = taosHashGetSize(tsDnodeVnodesHash);
int32_t count = taosHashGetSize(tsVnodesHash);
vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count);
}
......@@ -458,7 +455,7 @@ static void vnodeIncRef(void *ptNode) {
}
void *vnodeAcquire(int32_t vgId) {
SVnodeObj **ppVnode = taosHashGetCB(tsDnodeVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
if (ppVnode == NULL || *ppVnode == NULL) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
......@@ -497,7 +494,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
}
int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash);
while (taosHashIterNext(pIter)) {
SVnodeObj **pVnode = taosHashIterGet(pIter);
if (pVnode == NULL) continue;
......@@ -518,7 +515,7 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
void vnodeBuildStatusMsg(void *param) {
SStatusMsg *pStatus = param;
SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash);
while (taosHashIterNext(pIter)) {
SVnodeObj **pVnode = taosHashIterGet(pIter);
......@@ -547,7 +544,7 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) {
static void vnodeCleanUp(SVnodeObj *pVnode) {
// remove from hash, so new messages wont be consumed
taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
taosHashRemove(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
if (pVnode->status != TAOS_VN_STATUS_INIT) {
// it may be in updateing or reset state, then it shall wait
......@@ -561,9 +558,9 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
}
// stop replication module
if (pVnode->sync) {
void *sync = pVnode->sync;
pVnode->sync = NULL;
if (pVnode->sync > 0) {
int64_t sync = pVnode->sync;
pVnode->sync = -1;
syncStop(sync);
}
......
......@@ -15,13 +15,10 @@
#define _DEFAULT_SOURCE
#define _NON_BLOCKING_RETRIEVE 0
#include "os.h"
#include "tglobal.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tcache.h"
#include "query.h"
#include "trpc.h"
#include "tsdb.h"
......@@ -29,9 +26,9 @@
#include "vnodeInt.h"
#include "tqueue.h"
static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pReadMsg);
static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg);
static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg);
static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pRead);
static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId);
void vnodeInitReadFp(void) {
......@@ -44,16 +41,16 @@ void vnodeInitReadFp(void) {
// still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the
// request enters the queue
//
int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) {
int32_t vnodeProcessRead(void *param, SVReadMsg *pRead) {
SVnodeObj *pVnode = (SVnodeObj *)param;
int32_t msgType = pReadMsg->msgType;
int32_t msgType = pRead->msgType;
if (vnodeProcessReadMsgFp[msgType] == NULL) {
vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]);
return TSDB_CODE_VND_MSG_NOT_PROCESSED;
}
return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg);
return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead);
}
static int32_t vnodeCheckRead(void *param) {
......@@ -180,27 +177,27 @@ static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) {
pRsp->completed = true;
}
static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
void * pCont = pReadMsg->pCont;
int32_t contLen = pReadMsg->contLen;
SRspRet *pRet = &pReadMsg->rspRet;
static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
void * pCont = pRead->pCont;
int32_t contLen = pRead->contLen;
SRspRet *pRet = &pRead->rspRet;
SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont;
memset(pRet, 0, sizeof(SRspRet));
// qHandle needs to be freed correctly
if (pReadMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont;
if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pRead->pCont;
killQueryMsg->free = htons(killQueryMsg->free);
killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcHandle);
assert(pReadMsg->contLen > 0 && killQueryMsg->free == 1);
vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pRead->rpcHandle);
assert(pRead->contLen > 0 && killQueryMsg->free == 1);
void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle);
if (qhandle == NULL || *qhandle == NULL) {
vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle,
pReadMsg->rpcHandle);
pRead->rpcHandle);
} else {
assert(*qhandle == (void *)killQueryMsg->qhandle);
......@@ -242,9 +239,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
}
if (handle != NULL &&
vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle,
pReadMsg->rpcHandle);
pRead->rpcHandle);
pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
return pRsp->code;
......@@ -255,7 +252,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
if (handle != NULL) {
vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle);
code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcHandle);
code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle);
if (code != TSDB_CODE_SUCCESS) {
pRsp->code = code;
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
......@@ -264,7 +261,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
}
} else {
assert(pCont != NULL);
void **qhandle = (void **)pReadMsg->qhandle;
void **qhandle = (void **)pRead->qhandle;
vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
......@@ -276,14 +273,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
// build query rsp, the retrieve request has reached here already
if (buildRes) {
// update the connection info according to the retrieve connection
pReadMsg->rpcHandle = qGetResultRetrieveMsg(*qhandle);
assert(pReadMsg->rpcHandle != NULL);
pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle);
assert(pRead->rpcHandle != NULL);
vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle,
pReadMsg->rpcHandle);
pRead->rpcHandle);
// set the real rsp error code
pReadMsg->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcHandle);
pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle);
// NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
code = TSDB_CODE_QRY_HAS_RSP;
......@@ -308,16 +305,16 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
return code;
}
static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
void * pCont = pReadMsg->pCont;
SRspRet *pRet = &pReadMsg->rspRet;
static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
void * pCont = pRead->pCont;
SRspRet *pRet = &pRead->rspRet;
SRetrieveTableMsg *pRetrieve = pCont;
pRetrieve->free = htons(pRetrieve->free);
pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle,
pRetrieve->free, pReadMsg->rpcHandle);
pRetrieve->free, pRead->rpcHandle);
memset(pRet, 0, sizeof(SRspRet));
......@@ -348,8 +345,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
}
// register the qhandle to connect to quit query immediate if connection is broken
if (vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcHandle);
if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pRead->rpcHandle);
code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
qKillQuery(*handle);
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
......@@ -359,7 +356,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
bool freeHandle = true;
bool buildRes = false;
code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcHandle);
code = qRetrieveQueryResultInfo(*handle, &buildRes, pRead->rpcHandle);
if (code != TSDB_CODE_SUCCESS) {
// TODO handle malloc failure
pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
......@@ -370,7 +367,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
assert(buildRes == true);
#if _NON_BLOCKING_RETRIEVE
if (!buildRes) {
assert(pReadMsg->rpcHandle != NULL);
assert(pRead->rpcHandle != NULL);
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
return TSDB_CODE_QRY_NOT_READY;
......@@ -378,7 +375,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
#endif
// ahandle is the sqlObj pointer
code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcHandle);
code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pRead->rpcHandle);
}
// If qhandle is not added into vread queue, the query should be completed already or paused with error.
......
......@@ -61,10 +61,10 @@ int32_t vnodeReadVersion(SVnodeObj *pVnode) {
vError("vgId:%d, failed to read %s, version not found", pVnode->vgId, file);
goto PARSE_VER_ERROR;
}
pVnode->version = ver->valueint;
pVnode->version = (uint64_t)ver->valueint;
terrno = TSDB_CODE_SUCCESS;
vInfo("vgId:%d, read %s successfully, version:%" PRId64, pVnode->vgId, file, pVnode->version);
vInfo("vgId:%d, read %s successfully, version:%" PRIu64, pVnode->vgId, file, pVnode->version);
PARSE_VER_ERROR:
if (content != NULL) free(content);
......@@ -89,7 +89,7 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
char * content = calloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"version\": %" PRId64 "\n", pVnode->fversion);
len += snprintf(content + len, maxLen - len, " \"version\": %" PRIu64 "\n", pVnode->fversion);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
......@@ -98,6 +98,6 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
free(content);
terrno = 0;
vInfo("vgId:%d, successed to write %s, version:%" PRId64, pVnode->vgId, file, pVnode->fversion);
vInfo("vgId:%d, successed to write %s, version:%" PRIu64, pVnode->vgId, file, pVnode->fversion);
return TSDB_CODE_SUCCESS;
}
\ No newline at end of file
......@@ -19,7 +19,6 @@
#include "taoserror.h"
#include "tqueue.h"
#include "trpc.h"
#include "tutil.h"
#include "tsdb.h"
#include "twal.h"
#include "tsync.h"
......@@ -185,7 +184,7 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
SDropSTableMsg *pTable = pCont;
int32_t code = TSDB_CODE_SUCCESS;
int32_t code = TSDB_CODE_SUCCESS;
vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId);
......
......@@ -43,6 +43,7 @@ extern int32_t wDebugFlag;
typedef struct {
uint64_t version;
int64_t fileId;
int64_t rid;
int32_t vgId;
int32_t fd;
int32_t keep;
......
......@@ -78,7 +78,8 @@ void *walOpen(char *path, SWalCfg *pCfg) {
return NULL;
}
if (taosAddRef(tsWal.refId, pWal) != TSDB_CODE_SUCCESS) {
pWal->rid = taosAddRef(tsWal.refId, pWal);
if (pWal->rid < 0) {
walFreeObj(pWal);
return NULL;
}
......@@ -127,7 +128,7 @@ void walClose(void *handle) {
taosClose(pWal->fd);
if (!pWal->keep) {
if (pWal->keep != TAOS_WAL_KEEP) {
int64_t fileId = -1;
while (walGetNextFile(pWal, &fileId) >= 0) {
snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId);
......@@ -143,7 +144,7 @@ void walClose(void *handle) {
}
pthread_mutex_unlock(&pWal->mutex);
taosRemoveRef(tsWal.refId, pWal);
taosRemoveRef(tsWal.refId, pWal->rid);
}
static int32_t walInitObj(SWal *pWal) {
......@@ -185,7 +186,7 @@ static void walUpdateSeq() {
}
static void walFsyncAll() {
SWal *pWal = taosIterateRef(tsWal.refId, NULL);
SWal *pWal = taosIterateRef(tsWal.refId, 0);
while (pWal) {
if (walNeedFsync(pWal)) {
wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq);
......@@ -194,7 +195,7 @@ static void walFsyncAll() {
wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code));
}
}
pWal = taosIterateRef(tsWal.refId, pWal);
pWal = taosIterateRef(tsWal.refId, pWal->rid);
}
}
......
......@@ -41,7 +41,7 @@ int32_t walRenew(void *handle) {
wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name);
}
if (pWal->keep) {
if (pWal->keep == TAOS_WAL_KEEP) {
pWal->fileId = 0;
} else {
if (walGetNewFile(pWal, &pWal->fileId) != 0) pWal->fileId = 0;
......@@ -58,7 +58,7 @@ int32_t walRenew(void *handle) {
wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name);
}
if (!pWal->keep) {
if (pWal->keep != TAOS_WAL_KEEP) {
// remove the oldest wal file
int64_t oldFileId = -1;
if (walGetOldFile(pWal, pWal->fileId, WAL_FILE_NUM, &oldFileId) == 0) {
......@@ -144,12 +144,12 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) {
continue;
}
wDebug("vgId:%d, file:%s, restore success and keep it", pWal->vgId, walName);
wDebug("vgId:%d, file:%s, restore success", pWal->vgId, walName);
count++;
}
if (!pWal->keep) return TSDB_CODE_SUCCESS;
if (pWal->keep != TAOS_WAL_KEEP) return TSDB_CODE_SUCCESS;
if (count == 0) {
wDebug("vgId:%d, wal file not exist, renew it", pWal->vgId);
......@@ -173,7 +173,6 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) {
if (handle == NULL) return -1;
SWal *pWal = handle;
// for keep
if (*fileId == 0) *fileId = -1;
pthread_mutex_lock(&(pWal->mutex));
......@@ -311,7 +310,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
return code;
}
int64_t walGetVersion(twalh param) {
uint64_t walGetVersion(twalh param) {
SWal *pWal = param;
if (pWal == 0) return 0;
......
......@@ -37,7 +37,6 @@ int writeToQueue(void *pVnode, void *data, int type, void *pMsg) {
int main(int argc, char *argv[]) {
char path[128] = "/home/jhtao/test/wal";
int max = 3;
int level = 2;
int total = 5;
int rows = 10000;
......@@ -47,8 +46,6 @@ int main(int argc, char *argv[]) {
for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) {
tstrncpy(path, argv[++i], sizeof(path));
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
max = atoi(argv[++i]);
} else if (strcmp(argv[i], "-l")==0 && i < argc-1) {
level = atoi(argv[++i]);
} else if (strcmp(argv[i], "-r")==0 && i < argc-1) {
......@@ -66,7 +63,6 @@ int main(int argc, char *argv[]) {
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-p path]: wal file path default is:%s\n", path);
printf(" [-m max]: max wal files, default is:%d\n", max);
printf(" [-l level]: log level, default is:%d\n", level);
printf(" [-t total]: total wal files, default is:%d\n", total);
printf(" [-r rows]: rows of records per wal file, default is:%d\n", rows);
......@@ -82,7 +78,6 @@ int main(int argc, char *argv[]) {
SWalCfg walCfg;
walCfg.walLevel = level;
walCfg.wals = max;
walCfg.keep = keep;
pWal = walOpen(path, &walCfg);
......
#!/bin/bash
today=`date +"%Y%m%d"`
WORK_DIR=/home/ubuntu/pxiao/
PERFORMANCE_TEST_REPORT=$TDENGINE_DIR/tests/performance-test-report-$today.log
# Coloured Echoes #
function red_echo { echo -e "\033[31m$@\033[0m"; } #
function green_echo { echo -e "\033[32m$@\033[0m"; } #
function yellow_echo { echo -e "\033[33m$@\033[0m"; } #
function white_echo { echo -e "\033[1;37m$@\033[0m"; } #
# Coloured Printfs #
function red_printf { printf "\033[31m$@\033[0m"; } #
function green_printf { printf "\033[32m$@\033[0m"; } #
function yellow_printf { printf "\033[33m$@\033[0m"; } #
function white_printf { printf "\033[1;37m$@\033[0m"; } #
# Debugging Outputs #
function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } #
function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } #
function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } #
function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } #
function stopTaosd {
echo "Stop taosd"
systemctl stop taosd
snap stop tdengine
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
while [ -n "$PID" ]
do
pkill -TERM -x taosd
sleep 1
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
}
function buildTDengine {
echoInfo "Build TDengine"
cd $WORK_DIR/TDengine
git remote update > /dev/null
REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
LOCAL_COMMIT=`git rev-parse --short @`
echo " LOCAL: $LOCAL_COMMIT"
echo "REMOTE: $REMOTE_COMMIT"
if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
echo "repo up-to-date"
else
echo "repo need to pull"
git pull > /dev/null
LOCAL_COMMIT=`git rev-parse --short @`
cd debug
rm -rf *
cmake .. > /dev/null
make > /dev/null
make install
fi
}
function runQueryPerfTest {
nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taodperf/ > /dev/null 2>&1 &
echoInfo "Run Performance Test"
cd $WORK_DIR/TDengine/tests/pytest
python3 query/queryPerformance.py | tee -a $PERFORMANCE_TEST_REPORT
}
function sendReport {
echo "send report"
receiver="pxiao@taosdata.com"
mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
cd $TDENGINE_DIR
sed -i 's/\x1b\[[0-9;]*m//g' $PERFORMANCE_TEST_REPORT
BODY_CONTENT=`cat $PERFORMANCE_TEST_REPORT`
echo -e "to: ${receiver}\nsubject: Query Performace Report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
(cat - && uuencode $PERFORMANCE_TEST_REPORT performance-test-report-$today.log) | \
ssmtp "${receiver}" && echo "Report Sent!"
}
stopTaosd
buildTDengine
runQueryPerfTest
echoInfo "Send Report"
sendReport
echoInfo "End of Test"
......@@ -2043,6 +2043,7 @@ class ThreadStacks: # stack info for all threads
print("[{sf}] File {filename}, line {lineno}, in {name}".format(
sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name))
print(" {}".format(frame.line))
stackFrame += 1
print("-----> End of Thread Info ----->\n")
class ClientManager:
......
......@@ -161,7 +161,7 @@ python3 ./test.py -f stream/metric_1.py
python3 ./test.py -f stream/new.py
python3 ./test.py -f stream/stream1.py
python3 ./test.py -f stream/stream2.py
python3 ./test.py -f stream/parser.py
#python3 ./test.py -f stream/parser.py
python3 ./test.py -f stream/history.py
#alter table
......@@ -207,3 +207,20 @@ python3 test.py -f tools/taosdemo.py
python3 test.py -f subscribe/singlemeter.py
#python3 test.py -f subscribe/stability.py
python3 test.py -f subscribe/supertable.py
# update
python3 ./test.py -f update/allow_update.py
python3 ./test.py -f update/allow_update-0.py
python3 ./test.py -f update/append_commit_data.py
python3 ./test.py -f update/append_commit_last-0.py
python3 ./test.py -f update/append_commit_last.py
python3 ./test.py -f update/merge_commit_data.py
python3 ./test.py -f update/merge_commit_data-0.py
python3 ./test.py -f update/merge_commit_data2.py
python3 ./test.py -f update/merge_commit_data2_update0.py
python3 ./test.py -f update/merge_commit_last-0.py
python3 ./test.py -f update/merge_commit_last.py
# wal
python3 ./test.py -f wal/addOldWalTest.py
\ No newline at end of file
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册