diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index a3d3b035e2c4564acd34a71fe1c1490ddc25ec75..e85ade60e509373dd52a37e8774a6bca1c7b9d56 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1675,8 +1675,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); - pTableMetaInfo->pTableMeta = - (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, pTableMeta, size, tsTableMetaKeepTimer); + pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, + strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer); // todo handle out of memory case if (pTableMetaInfo->pTableMeta == NULL) { @@ -1879,7 +1879,8 @@ int tscProcessShowRsp(SSqlObj *pSql) { size_t size = 0; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsTableMetaKeepTimer); + pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size, + tsTableMetaKeepTimer); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); if (pQueryInfo->colList == NULL) { @@ -1949,9 +1950,8 @@ int tscProcessDropDbRsp(SSqlObj *UNUSED_PARAM(pSql)) { int tscProcessDropTableRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pTableMetaInfo->name); - if (pTableMeta == NULL) { - /* not in cache, abort */ + STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); + if (pTableMeta == NULL) { /* not in cache, abort */ return 0; } @@ -1975,7 +1975,7 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pTableMetaInfo->name); + STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); if (pTableMeta == NULL) { /* not in cache, abort */ return 0; } @@ -2125,7 +2125,7 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false); } - pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByName(tscCacheHandle, pTableMetaInfo->name); + pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); if (pTableMetaInfo->pTableMeta != NULL) { STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns, diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 201ace43de171025b902d30d043cea04b755c02a..f1d69fa261ced440841ea4c4a574c5ace3a00594 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -148,7 +148,7 @@ void taos_init_imp() { refreshTime = refreshTime < 10 ? 10 : refreshTime; if (tscCacheHandle == NULL) { - tscCacheHandle = taosCacheInit(refreshTime); + tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL); } tscDebug("client is initialized successfully"); diff --git a/src/inc/query.h b/src/inc/query.h index 5fd2ede034ebfaaf86eecce7a429c33996606027..af3a89682c8c931b4e3aa5db250bb5e082759bdd 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -20,6 +20,7 @@ extern "C" { #endif typedef void* qinfo_t; +typedef void (*_qinfo_free_fn_t)(void*); /** * create the qinfo object according to QueryTableMsg @@ -28,15 +29,13 @@ typedef void* qinfo_t; * @param qinfo * @return */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo); +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, _qinfo_free_fn_t fn, qinfo_t* qinfo); /** * Destroy QInfo object * @param qinfo qhandle - * @param fp destroy callback function, while the qhandle is destoried, invoke the fp - * @param param free callback params */ -void qDestroyQueryInfo(qinfo_t qinfo, void (*fp)(void*), void* param); +void qDestroyQueryInfo(qinfo_t qinfo); /** * the main query execution function, including query on both table and multitables, @@ -81,11 +80,9 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo); /** * kill current ongoing query and free query handle automatically * @param qinfo qhandle - * @param fp destroy callback function, while the qhandle is destoried, invoke the fp - * @param param free callback params * @return */ -int32_t qKillQuery(qinfo_t qinfo, void (*fp)(void*), void* param); +int32_t qKillQuery(qinfo_t qinfo); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 4fce62d8d554f719d783a902c61bf394f5f7ec0a..90c2ff9e66d01a7fc89f4a1d8861025bc7668e7b 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -135,7 +135,8 @@ typedef struct SVgObj { char dbName[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; int8_t inUse; int8_t accessState; - int8_t reserved0[5]; + int8_t status; + int8_t reserved0[4]; SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; int8_t reserved1[7]; int8_t updateEnd[1]; diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index dc3ddac8aa73f1725349509ba1d25f9ff6cbf3f0..a8abddc967b9d2f2e0186e30cd0d9f65a2898c30 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -387,7 +387,6 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { pAccess++; mnodeDecVgroupRef(pVgroup); } - } if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index f3d6a3d344c84cdd03b43300ed0e3e1d416ec50d..a1d4be93c66898cb883850ceff87cac120f143ef 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -68,7 +68,7 @@ int32_t mnodeInitProfile() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg); - tsMnodeConnCache = taosCacheInitWithCb(CONN_CHECK_TIME, mnodeFreeConn); + tsMnodeConnCache = taosCacheInitWithCb(TSDB_DATA_TYPE_BINARY, CONN_CHECK_TIME,false, mnodeFreeConn); return 0; } @@ -101,8 +101,8 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { tstrncpy(connObj.user, user, sizeof(connObj.user)); char key[10]; - sprintf(key, "%u", connId); - SConnObj *pConn = taosCachePut(tsMnodeConnCache, key, &connObj, sizeof(connObj), CONN_KEEP_TIME); + int32_t len = sprintf(key, "%u", connId); + SConnObj *pConn = taosCachePut(tsMnodeConnCache, key, len, &connObj, sizeof(connObj), CONN_KEEP_TIME); mDebug("connId:%d, is created, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return pConn; @@ -115,10 +115,10 @@ void mnodeReleaseConn(SConnObj *pConn) { SConnObj *mnodeAccquireConn(uint32_t connId, char *user, uint32_t ip, uint16_t port) { char key[10]; - sprintf(key, "%u", connId); + int32_t len = sprintf(key, "%u", connId); uint64_t expireTime = CONN_KEEP_TIME * 1000 + (uint64_t)taosGetTimestampMs(); - SConnObj *pConn = taosCacheUpdateExpireTimeByName(tsMnodeConnCache, key, expireTime); + SConnObj *pConn = taosCacheUpdateExpireTimeByName(tsMnodeConnCache, key, len, expireTime); if (pConn == NULL) { mError("connId:%d, is already destroyed, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return NULL; @@ -547,7 +547,7 @@ static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) { int32_t queryId = (int32_t)strtol(queryIdStr, NULL, 10); - SConnObj *pConn = taosCacheAcquireByName(tsMnodeConnCache, connIdStr); + SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, connIdStr, strlen(connIdStr)); if (pConn == NULL) { mError("connId:%s, failed to kill queryId:%d, conn not exist", connIdStr, queryId); return TSDB_CODE_MND_INVALID_CONN_ID; @@ -577,7 +577,7 @@ static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) { int32_t streamId = (int32_t)strtol(streamIdStr, NULL, 10); - SConnObj *pConn = taosCacheAcquireByName(tsMnodeConnCache, connIdStr); + SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, connIdStr, strlen(connIdStr)); if (pConn == NULL) { mError("connId:%s, failed to kill streamId:%d, conn not exist", connIdStr, streamId); return TSDB_CODE_MND_INVALID_CONN_ID; @@ -594,7 +594,7 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg) { if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS; SCMKillConnMsg *pKill = pMsg->rpcMsg.pCont; - SConnObj * pConn = taosCacheAcquireByName(tsMnodeConnCache, pKill->queryId); + SConnObj * pConn = taosCacheAcquireByKey(tsMnodeConnCache, pKill->queryId, strlen(pKill->queryId)); if (pConn == NULL) { mError("connId:%s, failed to kill, conn not exist", pKill->queryId); return TSDB_CODE_MND_INVALID_CONN_ID; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 06ef2cb452fb063f642b12c6b3c7c5cf44d1b673..12b434a5131fc5a3188e383f4e25b62262f4e5d1 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -65,7 +65,7 @@ int32_t mnodeInitShow() { mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mnodeProcessConnectMsg); mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mnodeProcessUseMsg); - tsMnodeShowCache = taosCacheInitWithCb(5, mnodeFreeShowObj); + tsMnodeShowCache = taosCacheInitWithCb(TSDB_DATA_TYPE_BINARY, 5, false, mnodeFreeShowObj); return 0; } @@ -365,9 +365,9 @@ static bool mnodeCheckShowFinished(SShowObj *pShow) { static bool mnodeAccquireShowObj(SShowObj *pShow) { char key[10]; - sprintf(key, "%d", pShow->index); + int32_t len = sprintf(key, "%d", pShow->index); - SShowObj *pSaved = taosCacheAcquireByName(tsMnodeShowCache, key); + SShowObj *pSaved = taosCacheAcquireByKey(tsMnodeShowCache, key, len); if (pSaved == pShow) { mDebug("%p, show is accquired from cache", pShow); return true; @@ -380,9 +380,9 @@ static void *mnodePutShowObj(SShowObj *pShow, int32_t size) { if (tsMnodeShowCache != NULL) { char key[10]; pShow->index = atomic_add_fetch_32(&tsShowObjIndex, 1); - sprintf(key, "%d", pShow->index); + int32_t len = sprintf(key, "%d", pShow->index); - SShowObj *newQhandle = taosCachePut(tsMnodeShowCache, key, pShow, size, 6); + SShowObj *newQhandle = taosCachePut(tsMnodeShowCache, key, len, pShow, size, 6); free(pShow); mDebug("%p, show is put into cache, index:%s", newQhandle, key); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 25d667c5cfa6bb71b6083e132ab847c843a2214b..c86d9ad2506c7995ac6f679056f2a987235e7680 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -38,6 +38,11 @@ #include "mnodeVgroup.h" #include "mnodePeer.h" +typedef enum { + TAOS_VG_STATUS_READY, + TAOS_VG_STATUS_DROPPING +} EVgroupStatus; + static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; @@ -252,7 +257,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o pNextV++; } - if (i == openVnodes) { + if (i == openVnodes && pVgroup->status == TAOS_VG_STATUS_READY) { mnodeSendCreateVgroupMsg(pVgroup, NULL); } @@ -835,6 +840,7 @@ void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { } static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) { + pVgroup->status = TAOS_VG_STATUS_DROPPING; // deleting mDebug("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index b09f34b562294e13f9ee7e9e728f1ad974f8c9b1..46e7fd45fac42418650ce0dfdb0b0abb8316b1ed 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -58,7 +58,7 @@ static void httpDestroyContext(void *data) { } bool httpInitContexts() { - tsHttpServer.contextCache = taosCacheInitWithCb(2, httpDestroyContext); + tsHttpServer.contextCache = taosCacheInitWithCb(TSDB_DATA_TYPE_BINARY, 2, false, httpDestroyContext); if (tsHttpServer.contextCache == NULL) { httpError("failed to init context cache"); return false; @@ -104,14 +104,14 @@ HttpContext *httpCreateContext(int32_t fd) { if (pContext == NULL) return NULL; char contextStr[16] = {0}; - snprintf(contextStr, sizeof(contextStr), "%p", pContext); + int32_t keySize = snprintf(contextStr, sizeof(contextStr), "%p", pContext); pContext->fd = fd; pContext->httpVersion = HTTP_VERSION_10; pContext->lastAccessTime = taosGetTimestampSec(); pContext->state = HTTP_CONTEXT_STATE_READY; - HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, contextStr, &pContext, sizeof(HttpContext *), 3); + HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, contextStr, keySize, &pContext, sizeof(HttpContext *), 3); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, item:%p", pContext, fd, ppContext); @@ -123,9 +123,9 @@ HttpContext *httpCreateContext(int32_t fd) { HttpContext *httpGetContext(void *ptr) { char contextStr[16] = {0}; - snprintf(contextStr, sizeof(contextStr), "%p", ptr); + int32_t len = snprintf(contextStr, sizeof(contextStr), "%p", ptr); - HttpContext **ppContext = taosCacheAcquireByName(tsHttpServer.contextCache, contextStr); + HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, contextStr, len); if (ppContext) { HttpContext *pContext = *ppContext; diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 9276637d0ee99001d3ec2fc9f50626398a12ee24..82666826bcbdace2c1d5a58c1cda82c89553fe4e 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -441,7 +441,7 @@ void httpJsonPairStatus(JsonBuf* buf, int code) { } else { httpJsonPair(buf, "status", 6, "error", 5); httpJsonItemToken(buf); - httpJsonPairIntVal(buf, "code", 4, code); + httpJsonPairIntVal(buf, "code", 4, code & 0XFFFF); httpJsonItemToken(buf); if (code == TSDB_CODE_MND_DB_NOT_SELECTED) { httpJsonPair(buf, "desc", 4, "failed to create database", 23); diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index de52e10f9a7d71590ad481770a382d88264c221e..f53aff7831619319c1600cb1f6a7605cb33ed1be 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -174,9 +174,9 @@ void httpSendErrorRespWithDesc(HttpContext *pContext, int errNo, char *desc) { } if (desc == NULL) { - httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 1000, httpMsg[errNo]); + httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 5000, httpMsg[errNo]); } else { - httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 1000, desc); + httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 5000, desc); } } @@ -184,7 +184,8 @@ void httpSendErrorResp(HttpContext *pContext, int errNo) { httpSendErrorRespWith void httpSendTaosdErrorResp(HttpContext *pContext, int errCode) { int httpCode = 400; - httpSendErrorRespImp(pContext, httpCode, "Bad Request", 1000, (char*)tstrerror(errCode)); + + httpSendErrorRespImp(pContext, httpCode, "Bad Request", errCode & 0XFFFF, (char*)tstrerror(errCode)); } void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { @@ -200,7 +201,7 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { } else {} } - httpSendErrorRespImp(pContext, httpCode, "Bad Request", 1000, temp); + httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_TSC_INVALID_SQL & 0XFFFF, temp); } void httpSendSuccResp(HttpContext *pContext, char *desc) { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 6c82386d8187410544219456368be0b399928207..a5009c2347a4a582984338b8257c015196d7b582 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -85,6 +85,7 @@ bool httpReadDataImp(HttpContext *pContext) { } else { httpError("context:%p, fd:%d, ip:%s, read from socket error:%d, close connect", pContext, pContext->fd, pContext->ipstr, errno); + httpReleaseContext(pContext); return false; } } else { @@ -153,6 +154,7 @@ static bool httpReadData(HttpContext *pContext) { int ret = httpCheckReadCompleted(pContext); if (ret == HTTP_CHECK_BODY_CONTINUE) { //httpDebug("context:%p, fd:%d, ip:%s, not finished yet, wait another event", pContext, pContext->fd, pContext->ipstr); + httpReleaseContext(pContext); return false; } else if (ret == HTTP_CHECK_BODY_SUCCESS){ httpDebug("context:%p, fd:%d, ip:%s, thread:%s, read size:%d, dataLen:%d", @@ -161,11 +163,13 @@ static bool httpReadData(HttpContext *pContext) { return true; } else { httpNotifyContextClose(pContext); + httpReleaseContext(pContext); return false; } } else { httpError("context:%p, fd:%d, ip:%s, failed to read http body, close connect", pContext, pContext->fd, pContext->ipstr); httpNotifyContextClose(pContext); + httpReleaseContext(pContext); return false; } } diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index 83602e1291646d2f574b2628569eb5a95fa8e2de..3a901167d522adb725537f8ed5834bd68e1b6cb4 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -33,9 +33,9 @@ void httpCreateSession(HttpContext *pContext, void *taos) { memset(&session, 0, sizeof(HttpSession)); session.taos = taos; session.refCount = 1; - snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); + int32_t len = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - pContext->session = taosCachePut(server->sessionCache, session.id, &session, sizeof(HttpSession), tsHttpSessionExpire); + pContext->session = taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire); // void *temp = pContext->session; // taosCacheRelease(server->sessionCache, (void **)&temp, false); @@ -57,9 +57,9 @@ static void httpFetchSessionImp(HttpContext *pContext) { pthread_mutex_lock(&server->serverMutex); char sessionId[HTTP_SESSION_ID_LEN]; - snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); + int32_t len = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - pContext->session = taosCacheAcquireByName(server->sessionCache, sessionId); + pContext->session = taosCacheAcquireByKey(server->sessionCache, sessionId, len); if (pContext->session != NULL) { atomic_add_fetch_32(&pContext->session->refCount, 1); httpDebug("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, sessionRef:%d", pContext, pContext->fd, @@ -115,7 +115,7 @@ void httpCleanUpSessions() { } bool httpInitSessions() { - tsHttpServer.sessionCache = taosCacheInitWithCb(5, httpDestroySession); + tsHttpServer.sessionCache = taosCacheInitWithCb(TSDB_DATA_TYPE_BINARY, 5, false, httpDestroySession); if (tsHttpServer.sessionCache == NULL) { httpError("failed to init session cache"); return false; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index a83072384595d96fc60cc57ff35b1502c4b83b49..d7d59230b65bff91d28f42e732a54acb176019f7 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -27,6 +27,7 @@ #include "tref.h" #include "tsdb.h" #include "tsqlfunction.h" +#include "query.h" struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); @@ -181,13 +182,13 @@ typedef struct SQueryRuntimeEnv { } SQueryRuntimeEnv; typedef struct SQInfo { - void* signature; - int32_t pointsInterpo; - int32_t code; // error code to returned to client - sem_t dataReady; - void* tsdb; - int32_t vgId; - + void* signature; + int32_t pointsInterpo; + int32_t code; // error code to returned to client + sem_t dataReady; + void* tsdb; + void* param; + int32_t vgId; STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; @@ -202,8 +203,9 @@ typedef struct SQInfo { * We later may refactor to remove this attribution by using another flag to denote * whether a multimeter query is completed or not. */ - int32_t tableIndex; - int32_t numOfGroupResultPages; + int32_t tableIndex; + int32_t numOfGroupResultPages; + _qinfo_free_fn_t fn; } SQInfo; #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 5a17e4e1e842b785538546ab0ee149f9b6e5dcc6..f33d739ba1f41e22c90ee33f6cc6133febbe0bee 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4069,7 +4069,7 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { return pFillCol; } -int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool isSTableQuery) { +int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery, void* freeParam, _qinfo_free_fn_t fn) { int32_t code = TSDB_CODE_SUCCESS; SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; @@ -4083,14 +4083,16 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; + pQInfo->param = freeParam; + pQInfo->fn = fn; pRuntimeEnv->pQuery = pQuery; - pRuntimeEnv->pTSBuf = param; + pRuntimeEnv->pTSBuf = pTsBuf; pRuntimeEnv->cur.vgroupIndex = -1; pRuntimeEnv->stableQuery = isSTableQuery; pRuntimeEnv->prevGroupId = INT32_MIN; - if (param != NULL) { + if (pTsBuf != NULL) { int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); } @@ -5697,8 +5699,7 @@ static bool isValidQInfo(void *param) { return (sig == (uint64_t)pQInfo); } - -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) { +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable, void* param, _qinfo_free_fn_t fn) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -5731,7 +5732,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ } // filter the qualified - if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { + if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable, param, fn)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -5894,7 +5895,8 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { return TSDB_CODE_SUCCESS; } -int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) { +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, void* param, _qinfo_free_fn_t fn, + qinfo_t* pQInfo) { assert(pQueryMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; @@ -5984,7 +5986,7 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi goto _over; } - code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery); + code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery, param, fn); _over: free(tagCond); @@ -6020,7 +6022,7 @@ static void doDestoryQueryInfo(SQInfo* pQInfo) { freeQInfo(pQInfo); } -void qDestroyQueryInfo(qinfo_t qHandle, void (*fp)(void*), void* param) { +void qDestroyQueryInfo(qinfo_t qHandle) { SQInfo* pQInfo = (SQInfo*) qHandle; if (!isValidQInfo(pQInfo)) { return; @@ -6030,11 +6032,15 @@ void qDestroyQueryInfo(qinfo_t qHandle, void (*fp)(void*), void* param) { qDebug("QInfo:%p dec refCount, value:%d", pQInfo, ref); if (ref == 0) { - doDestoryQueryInfo(pQInfo); + _qinfo_free_fn_t fn = pQInfo->fn; + void* param = pQInfo->param; - if (fp != NULL) { - fp(param); + doDestoryQueryInfo(pQInfo); + if (fn != NULL) { + assert(param != NULL); + fn(param); } + } } @@ -6048,7 +6054,7 @@ void qTableQuery(qinfo_t qinfo, void (*fp)(void*), void* param) { if (isQueryKilled(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - qDestroyQueryInfo(pQInfo, fp, param); + qDestroyQueryInfo(pQInfo); return; } @@ -6069,7 +6075,7 @@ void qTableQuery(qinfo_t qinfo, void (*fp)(void*), void* param) { } sem_post(&pQInfo->dataReady); - qDestroyQueryInfo(pQInfo, fp, param); + qDestroyQueryInfo(pQInfo); } int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { @@ -6162,7 +6168,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co return code; } -int32_t qKillQuery(qinfo_t qinfo, void (*fp)(void*), void* param) { +int32_t qKillQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; if (pQInfo == NULL || !isValidQInfo(pQInfo)) { @@ -6170,8 +6176,7 @@ int32_t qKillQuery(qinfo_t qinfo, void (*fp)(void*), void* param) { } setQueryKilled(pQInfo); - qDestroyQueryInfo(pQInfo, fp, param); - + qDestroyQueryInfo(pQInfo); return TSDB_CODE_SUCCESS; } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 3a9397502251e308069ecb65699fb5383c550432..e4f364d3d340a3407e01477f6342c1e553a152e8 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -641,7 +641,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { pConn->pRpc = pRpc; pConn->sid = sid; - pConn->tranId = (uint16_t)(rand() & 0xFFFF); + pConn->tranId = (uint16_t)(random() & 0xFFFF); pConn->ownId = htonl(pConn->sid); pConn->linkUid = (uint32_t)((int64_t)pConn + (int64_t)getpid()); pConn->spi = pRpc->spi; @@ -1529,10 +1529,10 @@ static void rpcAddRef(SRpcInfo *pRpc) static void rpcDecRef(SRpcInfo *pRpc) { if (atomic_sub_fetch_32(&pRpc->refCount, 1) == 0) { + rpcCloseConnCache(pRpc->pCache); taosHashCleanup(pRpc->hash); taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); - rpcCloseConnCache(pRpc->pCache); tfree(pRpc->connList); pthread_mutex_destroy(&pRpc->mutex); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index e650cef45cf14a351ff7986985df43b4d1112b71..4b9e977a1b741c6998b3c6bca8b7e7ec2cae6a27 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -415,11 +415,11 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { // Check maxTables if (pCfg->maxTables == -1) { - pCfg->maxTables = TSDB_DEFAULT_TABLES; + pCfg->maxTables = TSDB_DEFAULT_TABLES+1; } else { - if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) { + if (pCfg->maxTables - 1 < TSDB_MIN_TABLES || pCfg->maxTables - 1 > TSDB_MAX_TABLES) { tsdbError("vgId:%d invalid maxTables configuration! maxTables %d TSDB_MIN_TABLES %d TSDB_MAX_TABLES %d", - pCfg->tsdbId, pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); + pCfg->tsdbId, pCfg->maxTables - 1, TSDB_MIN_TABLES, TSDB_MAX_TABLES); goto _err; } } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index dafc7dbb1bcb2eab6fa5f8f6587cc322879512d1..9c9ac1699a38f1545de273b41792ecb32677b812 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -541,7 +541,7 @@ void tsdbRefTable(STable *pTable) { void tsdbUnRefTable(STable *pTable) { int32_t ref = T_REF_DEC(pTable); - tsdbDebug("unref table uid:%"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref); + tsdbTrace("unref table uid:%"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref); if (ref == 0) { // tsdbDebug("destory table name:%s uid:%"PRIu64", tid:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable)); diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 17b38238316bf74d67824b119b774a8cd5804860..cd3d0d436f68b25fdc9fa71175acaaea5b5f408c 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -24,6 +24,8 @@ extern "C" { #include "tref.h" #include "hash.h" +typedef void (*__cache_freeres_fn_t)(void*); + typedef struct SCacheStatis { int64_t missCount; int64_t hitCount; @@ -34,14 +36,15 @@ typedef struct SCacheStatis { typedef struct SCacheDataNode { uint64_t addedTime; // the added time when this element is added or updated into cache - uint64_t expiredTime; // expiredTime expiredTime when this element should be remove from cache + uint64_t lifespan; // expiredTime expiredTime when this element should be remove from cache uint64_t signature; uint32_t size; // allocated size for current SCacheDataNode - uint16_t keySize: 15; - bool inTrashCan: 1;// denote if it is in trash or not T_REF_DECLARE() - char *key; - char data[]; + uint16_t keySize: 15; // max key size: 32kb + bool inTrashCan: 1;// denote if it is in trash or not + int32_t extendFactor; // number of life span extend + char *key; + char data[]; } SCacheDataNode; typedef struct STrashElem { @@ -62,29 +65,32 @@ typedef struct { int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included. int64_t refreshTime; STrashElem * pTrash; - void * tmrCtrl; - void * pTimer; +// void * tmrCtrl; +// void * pTimer; SCacheStatis statistics; SHashObj * pHashTable; - _hash_free_fn_t freeFp; + __cache_freeres_fn_t freeFp; uint32_t numOfElemsInTrash; // number of element in trash uint8_t deleting; // set the deleting flag to stop refreshing ASAP. pthread_t refreshWorker; - + bool extendLifespan; // auto extend life span when one item is accessed. #if defined(LINUX) pthread_rwlock_t lock; #else - pthread_mutex_t lock; + pthread_mutex_t lock; #endif } SCacheObj; /** * initialize the cache object - * @param refreshTime refresh operation interval time, the maximum survival time when one element is expired and - * not referenced by other objects + * @param keyType key type + * @param refreshTimeInSeconds refresh operation interval time, the maximum survival time when one element is expired + * and not referenced by other objects + * @param extendLifespan auto extend lifespan, if accessed + * @param fn free resource callback function * @return */ -SCacheObj *taosCacheInit(int64_t refreshTimeInSeconds); +SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn); /** * initialize the cache object and set the free object callback function @@ -92,7 +98,7 @@ SCacheObj *taosCacheInit(int64_t refreshTimeInSeconds); * @param freeCb * @return */ -SCacheObj *taosCacheInitWithCb(int64_t refreshTimeInSeconds, void (*freeCb)(void *data)); +SCacheObj *taosCacheInitWithCb(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn); /** * add data into cache @@ -104,7 +110,7 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTimeInSeconds, void (*freeCb)(void * @param keepTime survival time in second * @return cached element */ -void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, size_t dataSize, int keepTimeInSeconds); +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int keepTimeInSeconds); /** * get data from cache @@ -112,22 +118,23 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz * @param key key * @return cached data or NULL */ -void *taosCacheAcquireByName(SCacheObj *pCacheObj, const char *key); +void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen); /** * update the expire time of data in cache * @param pCacheObj cache object * @param key key + * @param keyLen keyLen * @param expireTime new expire time of data * @return */ -void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, uint64_t expireTime); +void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, size_t keyLen, uint64_t expireTime); /** * Add one reference count for the exist data, and assign this data for a new owner. * The new owner needs to invoke the taosCacheRelease when it does not need this data anymore. - * This procedure is a faster version of taosCacheAcquireByName function, which avoids the sideeffect of the problem of - * the data is moved to trash, and taosCacheAcquireByName will fail to retrieve it again. + * This procedure is a faster version of taosCacheAcquireByKey function, which avoids the sideeffect of the problem of + * the data is moved to trash, and taosCacheAcquireByKey will fail to retrieve it again. * * @param handle * @param data @@ -148,8 +155,7 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data); * if it is referenced by other object, it will be remain in cache * @param handle cache object * @param data not the key, actually referenced data - * @param _remove force model, reduce the ref count and move the data into - * pTrash + * @param _remove force model, reduce the ref count and move the data into pTrash */ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove); diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 720741f0893086d4c9c3fb59bee0adfe4dc3f534..d763472a12dbff7ee7db6a2fc6b6a887e63a7465 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -119,7 +119,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo int32_t size = pNode->size; taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - uDebug("key:%s, is removed from cache, total:%" PRId64 " size:%d bytes", pNode->key, pCacheObj->totalSize, size); + uDebug("key:%p, is removed from cache,total:%" PRId64 ",size:%dbytes", pNode->key, pCacheObj->totalSize, size); if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); free(pNode); } @@ -167,7 +167,7 @@ static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode // update the timestamp information for updated key/value pNewNode->addedTime = taosGetTimestampMs(); - pNewNode->expiredTime = pNewNode->addedTime + duration; + pNewNode->lifespan = duration; T_REF_INC(pNewNode); @@ -224,8 +224,8 @@ static void doCleanupDataCache(SCacheObj *pCacheObj); */ static void* taosCacheRefresh(void *handle); -SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) { - if (refreshTime <= 0) { +SCacheObj *taosCacheInitWithCb(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn) { + if (refreshTimeInSeconds <= 0) { return NULL; } @@ -235,7 +235,7 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) return NULL; } - pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false); if (pCacheObj->pHashTable == NULL) { free(pCacheObj); uError("failed to allocate memory, reason:%s", strerror(errno)); @@ -243,10 +243,9 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) } // set free cache node callback function for hash table - // taosHashSetFreecb(pCacheObj->pHashTable, taosFreeNode); - - pCacheObj->freeFp = freeCb; - pCacheObj->refreshTime = refreshTime * 1000; + pCacheObj->freeFp = fn; + pCacheObj->refreshTime = refreshTimeInSeconds * 1000; + pCacheObj->extendLifespan = extendLifespan; if (__cache_lock_init(pCacheObj) != 0) { taosHashCleanup(pCacheObj->pHashTable); @@ -256,7 +255,7 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) return NULL; } - pthread_attr_t thattr; + pthread_attr_t thattr = {{0}}; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -266,19 +265,17 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data)) return pCacheObj; } -SCacheObj *taosCacheInit(int64_t refreshTime) { - return taosCacheInitWithCb(refreshTime, NULL); +SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn) { + return taosCacheInitWithCb(keyType, refreshTimeInSeconds, extendLifespan, fn); } -void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, size_t dataSize, int duration) { +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { SCacheDataNode *pNode; if (pCacheObj == NULL || pCacheObj->pHashTable == NULL) { return NULL; } - - size_t keyLen = strlen(key); - + __cache_wr_lock(pCacheObj); SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); SCacheDataNode * pOld = (pt != NULL) ? (*pt) : NULL; @@ -288,14 +285,14 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz if (NULL != pNode) { pCacheObj->totalSize += pNode->size; - uDebug("key:%s, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%" PRId64 ", size:%" PRId64 " bytes", - key, pNode, pNode->addedTime, pNode->expiredTime, pCacheObj->totalSize, dataSize); + uDebug("key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%" PRId64 ", size:%" PRId64 " bytes", + key, pNode, pNode->addedTime, (pNode->lifespan * pNode->extendFactor + pNode->addedTime), pCacheObj->totalSize, dataSize); } else { - uError("key:%s, failed to added into cache, out of memory", key); + uError("key:%p, failed to added into cache, out of memory", key); } } else { // old data exists, update the node pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, keyLen, pData, dataSize, duration * 1000L); - uDebug("key:%s, %p exist in cache, updated", key, pNode); + uDebug("key:%p, %p exist in cache, updated", key, pNode); } __cache_unlock(pCacheObj); @@ -303,57 +300,65 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz return (pNode != NULL) ? pNode->data : NULL; } -void *taosCacheAcquireByName(SCacheObj *pCacheObj, const char *key) { +void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) { if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { return NULL; } - - uint32_t keyLen = (uint32_t)strlen(key); - + __cache_rd_lock(pCacheObj); SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); + + int32_t ref = 0; if (ptNode != NULL) { - T_REF_INC(*ptNode); + ref = T_REF_INC(*ptNode); + + // if the remained life span is less then the (*ptNode)->lifeSpan, add up one lifespan + if (pCacheObj->extendLifespan) { + int64_t now = taosGetTimestampMs(); + + if ((now - (*ptNode)->addedTime) < (*ptNode)->lifespan * (*ptNode)->extendFactor) { + (*ptNode)->extendFactor += 1; + uDebug("key:%p extend life time to %"PRId64, key, (*ptNode)->lifespan * (*ptNode)->extendFactor + (*ptNode)->addedTime); + } + } } - __cache_unlock(pCacheObj); if (ptNode != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("key:%s, is retrieved from cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); + uDebug("key:%p, is retrieved from cache, %p refcnt:%d", key, (*ptNode), ref); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); - uDebug("key:%s, not in cache, retrieved failed", key); + uDebug("key:%p, not in cache, retrieved failed", key); } atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); return (ptNode != NULL) ? (*ptNode)->data : NULL; } -void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, uint64_t expireTime) { +void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, const char *key, size_t keyLen, uint64_t expireTime) { if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { return NULL; } - - uint32_t keyLen = (uint32_t)strlen(key); - + __cache_rd_lock(pCacheObj); SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); if (ptNode != NULL) { T_REF_INC(*ptNode); - (*ptNode)->expiredTime = expireTime; + (*ptNode)->extendFactor += 1; +// (*ptNode)->lifespan = expireTime; } __cache_unlock(pCacheObj); if (ptNode != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("key:%s, expireTime is updated in cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); + uDebug("key:%p, expireTime is updated in cache, %p refcnt:%d", key, (*ptNode), T_REF_VAL_GET(*ptNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); - uDebug("key:%s, not in cache, retrieved failed", key); + uDebug("key:%p, not in cache, retrieved failed", key); } atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); @@ -373,7 +378,17 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { int32_t ref = T_REF_INC(ptNode); uDebug("%p acquired by data in cache, refcnt:%d", ptNode, ref) - + + // if the remained life span is less then the (*ptNode)->lifeSpan, add up one lifespan + if (pCacheObj->extendLifespan) { + int64_t now = taosGetTimestampMs(); + + if ((now - ptNode->addedTime) < ptNode->lifespan * ptNode->extendFactor) { + ptNode->extendFactor += 1; + uDebug("key:%p extend life time to %"PRId64, ptNode, ptNode->lifespan * ptNode->extendFactor + ptNode->addedTime); + } + } + // the data if referenced by at least one object, so the reference count must be greater than the value of 2. assert(ref >= 2); return data; @@ -408,21 +423,27 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); - if (pNode->signature != (uint64_t)pNode) { - uError("%p release invalid cache data", pNode); + uError("key:%p, release invalid cache data", pNode); return; } *data = NULL; - int32_t ref = T_REF_DEC(pNode); - uDebug("key:%s, is released, %p refcnt:%d", pNode->key, pNode, ref); + int16_t ref = T_REF_DEC(pNode); + uDebug("%p data released, refcnt:%d", pNode, ref); - if (_remove) { + if (_remove && (!pNode->inTrashCan)) { __cache_wr_lock(pCacheObj); - // pNode may be released immediately by other thread after the reference count of pNode is set to 0, - // So we need to lock it in the first place. - taosCacheMoveToTrash(pCacheObj, pNode); + + if (T_REF_VAL_GET(pNode) == 0) { + // remove directly, if not referenced by other users + taosCacheReleaseNode(pCacheObj, pNode); + } else { + // pNode may be released immediately by other thread after the reference count of pNode is set to 0, + // So we need to lock it in the first place. + taosCacheMoveToTrash(pCacheObj, pNode); + } + __cache_unlock(pCacheObj); } } @@ -473,11 +494,11 @@ SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char * memcpy(pNewNode->key, key, keyLen); - pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); - pNewNode->expiredTime = pNewNode->addedTime + duration; - - pNewNode->signature = (uint64_t)pNewNode; - pNewNode->size = (uint32_t)totalSize; + pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); + pNewNode->lifespan = duration; + pNewNode->extendFactor = 1; + pNewNode->signature = (uint64_t)pNewNode; + pNewNode->size = (uint32_t)totalSize; return pNewNode; } @@ -501,7 +522,7 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pNode->inTrashCan = true; pCacheObj->numOfElemsInTrash++; - uDebug("key:%s, %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash); + uDebug("key:%p, %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash); } void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) { @@ -522,7 +543,11 @@ void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) { } pElem->pData->signature = 0; - if (pCacheObj->freeFp) pCacheObj->freeFp(pElem->pData->data); + if (pCacheObj->freeFp) { + pCacheObj->freeFp(pElem->pData->data); + } + + uError("-------------------free obj:%p", pElem->pData); free(pElem->pData); free(pElem); } @@ -549,7 +574,7 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { } if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { - uDebug("key:%s, %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData, + uDebug("key:%p, %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData, pCacheObj->numOfElemsInTrash - 1); STrashElem *p = pElem; @@ -573,7 +598,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { if (T_REF_VAL_GET(pNode) <= 0) { taosCacheReleaseNode(pCacheObj, pNode); } else { - uDebug("key:%s, will not remove from cache, refcnt:%d", pNode->key, T_REF_VAL_GET(pNode)); + uDebug("key:%p, will not remove from cache, refcnt:%d", pNode->key, T_REF_VAL_GET(pNode)); } } taosHashDestroyIter(pIter); @@ -613,27 +638,32 @@ void* taosCacheRefresh(void *handle) { // reset the count value count = 0; - size_t num = taosHashGetSize(pCacheObj->pHashTable); - if (num == 0) { + size_t elemInHash = taosHashGetSize(pCacheObj->pHashTable); + if (elemInHash + pCacheObj->numOfElemsInTrash == 0) { continue; } - uint64_t expiredTime = taosGetTimestampMs(); pCacheObj->statistics.refreshCount++; - SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); + // refresh data in hash table + if (elemInHash > 0) { + int64_t expiredTime = taosGetTimestampMs(); - __cache_wr_lock(pCacheObj); - while (taosHashIterNext(pIter)) { - SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); - if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); + SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); + + __cache_wr_lock(pCacheObj); + while (taosHashIterNext(pIter)) { + SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); + if ((pNode->addedTime + pNode->lifespan * pNode->extendFactor) <= expiredTime && T_REF_VAL_GET(pNode) <= 0) { + taosCacheReleaseNode(pCacheObj, pNode); + } } - } - __cache_unlock(pCacheObj); + __cache_unlock(pCacheObj); + + taosHashDestroyIter(pIter); + } - taosHashDestroyIter(pIter); taosTrashCanEmpty(pCacheObj, false); } diff --git a/src/util/src/tmem.c b/src/util/src/tmem.c index ec5f90990b4d84a92f898656f835c71a976676fc..9c512ad8dc5938626d983d8263271f060abdc78c 100644 --- a/src/util/src/tmem.c +++ b/src/util/src/tmem.c @@ -193,7 +193,7 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3 return malloc_detect_leak(size, file, line); } - SMemBlock* blk = ((char*)ptr) - sizeof(SMemBlock); + SMemBlock* blk = (SMemBlock *)((char*)ptr) - sizeof(SMemBlock); if (blk->magic != MEMBLK_MAGIC) { if (fpAllocLog != NULL) { fprintf(fpAllocLog, "%s:%d: memory is allocated by default allocator.\n", file, line); diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp index 5762d5700bbb1945490846dcfaaf0e12f0fbaa27..43ac689ff47745cb2884455e39581f19586c7e7e 100644 --- a/src/util/tests/cacheTest.cpp +++ b/src/util/tests/cacheTest.cpp @@ -19,12 +19,12 @@ int32_t tsMaxMeterConnections = 200; // test cache TEST(testCase, client_cache_test) { const int32_t REFRESH_TIME_IN_SEC = 2; - SCacheObj* tscCacheHandle = taosCacheInit(REFRESH_TIME_IN_SEC); + SCacheObj* tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL); const char* key1 = "test1"; char data1[] = "test11"; - char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1)+1, 1); + char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, strlen(key1), data1, strlen(data1)+1, 1); sleep(REFRESH_TIME_IN_SEC+1); printf("obj is still valid: %s\n", cachedObj); @@ -33,7 +33,7 @@ TEST(testCase, client_cache_test) { taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); /* the object is cleared by cache clean operation */ - cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2)+1, 20); + cachedObj = (char*) taosCachePut(tscCacheHandle, key1, strlen(key1), data2, strlen(data2)+1, 20); printf("after updated: %s\n", cachedObj); printf("start to remove data from cache\n"); @@ -43,32 +43,32 @@ TEST(testCase, client_cache_test) { const char* key3 = "test2"; const char* data3 = "kkkkkkk"; - char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3) + 1, 1); + char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, strlen(key3), data3, strlen(data3) + 1, 1); printf("%s\n", cachedObj2); taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); sleep(3); - char* d = (char*) taosCacheAcquireByName(tscCacheHandle, key3); + char* d = (char*) taosCacheAcquireByKey(tscCacheHandle, key3, strlen(key3)); // assert(d == NULL); char key5[] = "test5"; char data5[] = "data5kkkkk"; - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data5, strlen(data5) + 1, 20); const char* data6= "new Data after updated"; taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data6, strlen(data6) + 1, 20); printf("%s\n", cachedObj2); taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); const char* data7 = "add call update procedure"; - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data7, strlen(data7) + 1, 20); printf("%s\n=======================================\n\n", cachedObj2); - char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5); + char* cc = (char*) taosCacheAcquireByKey(tscCacheHandle, key5, strlen(key5)); taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); taosCacheRelease(tscCacheHandle, (void**) &cc, false); @@ -76,7 +76,7 @@ TEST(testCase, client_cache_test) { const char* data8 = "ttft"; const char* key6 = "key6"; - char* ft = (char*) taosCachePut(tscCacheHandle, key6, data8, strlen(data8), 20); + char* ft = (char*) taosCachePut(tscCacheHandle, key6, strlen(key6), data8, strlen(data8), 20); taosCacheRelease(tscCacheHandle, (void**) &ft, false); /** @@ -85,7 +85,7 @@ TEST(testCase, client_cache_test) { uint64_t startTime = taosGetTimestampUs(); printf("Cache Performance Test\nstart time:%" PRIu64 "\n", startTime); for(int32_t i=0; i<1000; ++i) { - char* dd = (char*) taosCacheAcquireByName(tscCacheHandle, key6); + char* dd = (char*) taosCacheAcquireByKey(tscCacheHandle, key6, strlen(key6)); if (dd != NULL) { // printf("get the data\n"); } else { @@ -105,7 +105,7 @@ TEST(testCase, client_cache_test) { TEST(testCase, cache_resize_test) { const int32_t REFRESH_TIME_IN_SEC = 2; - auto* pCache = taosCacheInit(REFRESH_TIME_IN_SEC); + auto* pCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, false, NULL); char key[256] = {0}; char data[1024] = "abcdefghijk"; @@ -116,7 +116,7 @@ TEST(testCase, cache_resize_test) { for(int32_t i = 0; i < num; ++i) { int32_t len = sprintf(key, "abc_%7d", i); - taosCachePut(pCache, key, data, len, 3600); + taosCachePut(pCache, key, strlen(key), data, len, 3600); } uint64_t endTime = taosGetTimestampUs(); @@ -125,7 +125,7 @@ TEST(testCase, cache_resize_test) { startTime = taosGetTimestampUs(); for(int32_t i = 0; i < num; ++i) { int32_t len = sprintf(key, "abc_%7d", i); - void* k = taosCacheAcquireByName(pCache, key); + void* k = taosCacheAcquireByKey(pCache, key, len); assert(k != 0); } endTime = taosGetTimestampUs(); diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index e428eae688505f8930e0f981cb02326fea981752..76e53f3962ed55cf1a3aee875d10b83b22ac9c37 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -53,6 +53,7 @@ typedef struct { STsdbCfg tsdbCfg; SSyncCfg syncCfg; SWalCfg walCfg; + void *qHandlePool; // query handle pool char *rootDir; char db[TSDB_DB_NAME_LEN]; } SVnodeObj; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index f71f6adefbe3923aa877bdbc0ad3a93ffb5a8cc8..192998c8a6060a58ed91075b3ab9978592a2d2f0 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -15,19 +15,22 @@ #define _DEFAULT_SOURCE #include "os.h" + +#include "tcache.h" +#include "cJSON.h" +#include "dnode.h" #include "hash.h" #include "taoserror.h" #include "taosmsg.h" -#include "tutil.h" +#include "tglobal.h" #include "trpc.h" #include "tsdb.h" #include "ttime.h" #include "ttimer.h" -#include "cJSON.h" -#include "tglobal.h" -#include "dnode.h" +#include "tutil.h" #include "vnode.h" #include "vnodeInt.h" +#include "query.h" #define TSDB_VNODE_VERSION_CONTENT_LEN 31 @@ -43,6 +46,7 @@ static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uin static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); static void vnodeNotifyRole(void *ahandle, int8_t role); static void vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); +static void vnodeFreeqHandle(void* phandle); static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; @@ -279,6 +283,9 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { if (pVnode->role == TAOS_SYNC_ROLE_MASTER) cqStart(pVnode->cq); + const int32_t REFRESH_HANDLE_INTERVAL = 2; // every 2 seconds, rfresh handle pool + pVnode->qHandlePool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, vnodeFreeqHandle); + pVnode->events = NULL; pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); @@ -848,12 +855,12 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) { goto PARSE_OVER; } - cJSON *version = cJSON_GetObjectItem(root, "version"); - if (!version || version->type != cJSON_Number) { + cJSON *ver = cJSON_GetObjectItem(root, "version"); + if (!ver || ver->type != cJSON_Number) { vError("vgId:%d, failed to read vnode version, version not found", pVnode->vgId); goto PARSE_OVER; } - pVnode->version = version->valueint; + pVnode->version = ver->valueint; terrno = TSDB_CODE_SUCCESS; vInfo("vgId:%d, read vnode version successfully, version:%" PRId64, pVnode->vgId, pVnode->version); @@ -864,3 +871,12 @@ PARSE_OVER: if(fp) fclose(fp); return terrno; } + +void vnodeFreeqHandle(void *qHandle) { + void** handle = qHandle; + if (handle == NULL || *handle == NULL) { + return; + } + + qKillQuery(*handle); +} \ No newline at end of file diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 0c08c77e329828396e7c6760814f15e247c8977d..6b157a036752c4e260707a902a778b90281c1b44 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -15,20 +15,21 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosmsg.h" + +#include "tglobal.h" #include "taoserror.h" -#include "tqueue.h" +#include "taosmsg.h" +#include "tcache.h" +#include "query.h" #include "trpc.h" #include "tsdb.h" -#include "twal.h" -#include "tdataformat.h" #include "vnode.h" #include "vnodeInt.h" -#include "query.h" static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SReadMsg *pReadMsg); static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg); static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg); +static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId); void vnodeInitReadFp(void) { vnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg; @@ -58,19 +59,6 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); } -// notify connection(handle) that current qhandle is created, if current connection from -// client is broken, the query needs to be killed immediately. -static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) { - SRetrieveTableMsg* killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg)); - killQueryMsg->qhandle = htobe64((uint64_t) qhandle); - killQueryMsg->free = htons(1); - killQueryMsg->header.vgId = htonl(vgId); - killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); - - vDebug("QInfo:%p register qhandle to connect:%p", qhandle, handle); - return rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg)); -} - static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { void * pCont = pReadMsg->pCont; int32_t contLen = pReadMsg->contLen; @@ -88,18 +76,25 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { vWarn("QInfo:%p connection %p broken, kill query", (void*)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle); assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1); - // this message arrived here by means of the query message, so release the vnode is necessary - qKillQuery((qinfo_t) killQueryMsg->qhandle, vnodeRelease, pVnode); - vnodeRelease(pVnode); + // this message arrived here by means of the *query* message, so release the vnode is necessary + void** qhandle = taosCacheAcquireByKey(pVnode->qHandlePool, (void*) &killQueryMsg->qhandle, sizeof(killQueryMsg->qhandle)); + if (qhandle == NULL || *qhandle == NULL) { // todo handle invalid qhandle error + } else { +// qKillQuery((qinfo_t) killQueryMsg->qhandle); + taosCacheRelease(pVnode->qHandlePool, (void**) &qhandle, true); + } + + vnodeRelease(pVnode); return TSDB_CODE_TSC_QUERY_CANCELLED; // todo change the error code } int32_t code = TSDB_CODE_SUCCESS; qinfo_t pQInfo = NULL; + void** handle = NULL; if (contLen != 0) { - code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo); + code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, vnodeRelease, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->qhandle = htobe64((uint64_t) (pQInfo)); @@ -116,13 +111,15 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; // NOTE: there two refcount, needs to kill twice, todo refactor - qKillQuery(pQInfo, vnodeRelease, pVnode); - qKillQuery(pQInfo, vnodeRelease, pVnode); + // query has not been put into qhandle pool, kill it directly. + qKillQuery(pQInfo); + qKillQuery(pQInfo); return pRsp->code; } - vTrace("vgId:%d, QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo); + handle = taosCachePut(pVnode->qHandlePool, pQInfo, sizeof(pQInfo), &pQInfo, sizeof(pQInfo), tsShellActivityTimer * 2); + assert(*handle == pQInfo); } else { assert(pQInfo == NULL); vnodeRelease(pVnode); @@ -138,6 +135,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (pQInfo != NULL) { qTableQuery(pQInfo, vnodeRelease, pVnode); // do execute query + taosCacheRelease(pVnode->qHandlePool, (void**) &handle, false); } return code; @@ -152,10 +150,18 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRetrieve->free = htons(pRetrieve->free); memset(pRet, 0, sizeof(SRspRet)); + int32_t ret = 0; + + void** handle = taosCacheAcquireByKey(pVnode->qHandlePool, &pQInfo, sizeof(pQInfo)); + if (handle == NULL || *handle != pQInfo) { + ret = TSDB_CODE_QRY_INVALID_QHANDLE; + } if (pRetrieve->free == 1) { vDebug("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, pQInfo); - int32_t ret = qKillQuery(pQInfo, vnodeRelease, pVnode); + + taosCacheRelease(pVnode->qHandlePool, handle, true); +// int32_t ret = qKillQuery(pQInfo); pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); pRet->len = sizeof(SRetrieveTableRsp); @@ -184,10 +190,23 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRet->qhandle = pQInfo; code = TSDB_CODE_VND_ACTION_NEED_REPROCESSED; } else { // no further execution invoked, release the ref to vnode - qDestroyQueryInfo(pQInfo, vnodeRelease, pVnode); + taosCacheRelease(pVnode->qHandlePool, (void**) &handle, true); +// qDestroyQueryInfo(pQInfo); } } - vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, pQInfo); return code; } + +// notify connection(handle) that current qhandle is created, if current connection from +// client is broken, the query needs to be killed immediately. +int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) { + SRetrieveTableMsg* killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg)); + killQueryMsg->qhandle = htobe64((uint64_t) qhandle); + killQueryMsg->free = htons(1); + killQueryMsg->header.vgId = htonl(vgId); + killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); + + vDebug("QInfo:%p register qhandle to connect:%p", qhandle, handle); + return rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg)); +} \ No newline at end of file diff --git a/tests/pytest/import_merge/importDataLastSub.py b/tests/pytest/import_merge/importDataLastSub.py index 5c2069c90f52b4c786739f4d58e1ce41185c0f93..bfcad2d252cccb2404b3989c474310e0a19afe2e 100644 --- a/tests/pytest/import_merge/importDataLastSub.py +++ b/tests/pytest/import_merge/importDataLastSub.py @@ -32,6 +32,7 @@ class TDTestCase: tdDnodes.stop(1) tdDnodes.deploy(1) tdDnodes.start(1) + tdLog.sleep(5) tdSql.execute('reset query cache') tdSql.execute('drop database if exists db') @@ -61,6 +62,7 @@ class TDTestCase: tdDnodes.stop(1) tdLog.sleep(5) tdDnodes.start(1) + tdLog.sleep(5) tdLog.info("================= step5") tdLog.info("import 10 data totally repetitive") diff --git a/tests/pytest/query/queryMetaData.py b/tests/pytest/query/queryMetaData.py index 8fb9d9bf3a7a1648607f7943ac807a2ed69d8222..7b95e4a81cb3807062050099008aa3f73fbb4dab 100755 --- a/tests/pytest/query/queryMetaData.py +++ b/tests/pytest/query/queryMetaData.py @@ -22,7 +22,7 @@ class MetadataQuery: def initConnection(self): self.tables = 100000 self.records = 10 - self.numOfTherads = 10 + self.numOfTherads = 20 self.ts = 1537146000000 self.host = "127.0.0.1" self.user = "root" @@ -55,10 +55,10 @@ class MetadataQuery: def createTablesAndInsertData(self, threadID): cursor = self.connectDB() - cursor.execute("use test") - base = threadID * self.tables + cursor.execute("use test") tablesPerThread = int (self.tables / self.numOfTherads) + base = threadID * tablesPerThread for i in range(tablesPerThread): cursor.execute( '''create table t%d using meters tags( @@ -75,12 +75,11 @@ class MetadataQuery: (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100)) - for j in range(self.records): - cursor.execute( - "insert into t%d values(%d, %d)" % - (base + i + 1, self.ts + j, j)) - cursor.close() - self.conn.close() + + cursor.execute( + "insert into t%d values(%d, 1) (%d, 2) (%d, 3) (%d, 4) (%d, 5)" % + (base + i + 1, self.ts + 1, self.ts + 2, self.ts + 3, self.ts + 4, self.ts + 5)) + cursor.close() def queryData(self, query): cursor = self.connectDB() @@ -108,12 +107,17 @@ if __name__ == '__main__': print( "================= Create %d tables and insert %d records into each table =================" % (t.tables, t.records)) - startTime = datetime.now() + startTime = datetime.now() + threads = [] for i in range(t.numOfTherads): thread = threading.Thread( target=t.createTablesAndInsertData, args=(i,)) thread.start() - thread.join() + threads.append(thread) + + for th in threads: + th.join() + endTime = datetime.now() diff = (endTime - startTime).seconds print( diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh index 8ce1fd1294ffca8c5aaf2d036f953ae16e2a4952..eada5f67f7dae0d9395d869cfd0e368bb5d71f5a 100755 --- a/tests/pytest/regressiontest.sh +++ b/tests/pytest/regressiontest.sh @@ -137,6 +137,7 @@ python3 ./test.py -f query/filterFloatAndDouble.py python3 ./test.py -f query/filterOtherTypes.py python3 ./test.py -f query/queryError.py python3 ./test.py -f query/querySort.py +python3 ./test.py -f query/queryJoin.py #stream python3 ./test.py -f stream/stream1.py diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 5b35563e1bcb041fdcb2389c4609d91b3fa71e1a..a9da8e5671a588409954735f79c7323136aa5c4c 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -81,7 +81,7 @@ if __name__ == "__main__": else: toBeKilled = "valgrind.bin" - killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -HUP " % toBeKilled + killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -HUP > /dev/null 2>&1" % toBeKilled psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled processID = subprocess.check_output(psCmd, shell=True) @@ -91,8 +91,17 @@ if __name__ == "__main__": time.sleep(1) processID = subprocess.check_output(psCmd, shell=True) - fuserCmd = "fuser -k -n tcp 6030" - os.system(fuserCmd) + for port in range(6030, 6041): + usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port + processID = subprocess.check_output(usePortPID, shell=True) + + if processID: + killCmd = "kill -9 %s" % processID + os.system(killCmd) + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if valgrind: + time.sleep(2) tdLog.info('stop All dnodes') sys.exit(0) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index e24af473f38274d3a1122362d037fc00099c1528..370af1ba136c0a24ae5c8e89f87291fa2a624fc0 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -251,11 +251,16 @@ class TDDnode: psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -INT %s" % processID + killCmd = "kill -INT %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") + for port in range(6030, 6041): + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if self.valgrind: + time.sleep(2) self.running = 0 tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) @@ -272,11 +277,16 @@ class TDDnode: psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s" % processID + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") + for port in range(6030, 6041): + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if self.valgrind: + time.sleep(2) self.running = 0 tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) @@ -325,7 +335,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s" % processID + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -334,7 +344,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s" % processID + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -440,7 +450,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s" % processID + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -449,7 +459,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s" % processID + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim index dac7552edf710952ef6539e8ec74f57419df9bda..bf2a3b3b58a9469c18ec17c3c27b4fa45ff9813f 100644 --- a/tests/script/general/http/grafana.sim +++ b/tests/script/general/http/grafana.sim @@ -54,43 +54,43 @@ print =============== step2 - login system_content curl 127.0.0.1:6020/grafana/ print 1-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/grafana/xx print 2-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/grafana/login/xx/xx/ print 3-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"mnode invalid user"}@ then +if $system_content != @{"status":"error","code":849,"desc":"mnode invalid user"}@ then return -1 endi system_content curl 127.0.0.1:6020/grafana/root/1/123/1/1/3 print 4-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:6020/grafana/login/1/root/1/ print 5-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"mnode invalid user"}@ then +if $system_content != @{"status":"error","code":849,"desc":"mnode invalid user"}@ then return -1 endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:6020/grafana/root/1/login print 6-> $system_content -if $system_content != @{"status":"error","code":1010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:6020/grafana/root/1/login print 7-> $system_content -if $system_content != @{"status":"error","code":1010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then return -1 endi diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim index 5ee8bde46fd38aa99508673a7dded5bef3c630a2..7f39a9c74824b2696c59d1b8c179e776bb87214d 100644 --- a/tests/script/general/http/restful.sim +++ b/tests/script/general/http/restful.sim @@ -77,7 +77,7 @@ endi system_content curl 127.0.0.1:6020/rest/login/u2/aabcd_1234 print curl 127.0.0.1:6020/rest/login/u2/abcd_1234 -----> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index aa667daee48b10b8759a55def48eb6ac6dfff44e..60ffa4cb2814ae7bf495b87b05a2f41197d36ef0 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -14,57 +14,57 @@ print =============== step1 - login system_content curl 127.0.0.1:6020/rest/ print 1-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/rest/xx print 2-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/rest/login print 3-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi #4 system_content curl 127.0.0.1:6020/rest/login/root print 4-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/rest/login/root/123 print 5-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi system_content curl 127.0.0.1:6020/rest/login/root/123/1/1/3 print 6-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:6020/rest/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi #8 system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:6020/rest/login/root/1 print 8-> $system_content -if $system_content != @{"status":"error","code":1010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:6020/rest/login/root/1 print 9-> $system_content -if $system_content != @{"status":"error","code":1010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then return -1 endi @@ -93,40 +93,40 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 127.0.0.1:6020/rest/sql print 13-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"mnode database aleady exist"}@ then +if $system_content != @{"status":"error","code":897,"desc":"mnode database aleady exist"}@ then return -1 endi #14 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '' 127.0.0.1:6020/rest/sql print 14-> $system_content -if $system_content != @{"status":"error","code":1012,"desc":"no sql input"}@ then +if $system_content != @{"status":"error","code":5012,"desc":"no sql input"}@ then return -1 endi #system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'use d1' 127.0.0.1:6020/rest/sql #print 15-> $system_content -#if $system_content != @{"status":"error","code":1017,"desc":"no need to execute use db cmd"}@ then +#if $system_content != @{"status":"error","code":5017,"desc":"no need to execute use db cmd"}@ then #if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then # return -1 #endi #system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' use d1' 127.0.0.1:6020/rest/sql #print 16-> $system_content -#if $system_content != @{"status":"error","code":1017,"desc":"no need to execute use db cmd"}@ then +#if $system_content != @{"status":"error","code":5017,"desc":"no need to execute use db cmd"}@ then # return -1 #endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' used1' 127.0.0.1:6020/rest/sql print 17-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"invalid SQL: invalid SQL: syntax error near 'used1'"}@ then +if $system_content != @{"status":"error","code":512,"desc":"invalid SQL: invalid SQL: syntax error near 'used1'"}@ then return -1 endi #18 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' show tables;' 127.0.0.1:6020/rest/sql print 18-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"mnode db not selected"}@ then +if $system_content != @{"status":"error","code":896,"desc":"mnode db not selected"}@ then return -1 endi @@ -147,7 +147,7 @@ print =============== step3 - db system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1;' 127.0.0.1:6020/rest/sql print 21-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"mnode invalid table name"}@ then +if $system_content != @{"status":"error","code":866,"desc":"mnode invalid table name"}@ then return -1 endi diff --git a/tests/script/general/http/telegraf.sim b/tests/script/general/http/telegraf.sim index d9f3340f06fa80dad2291ceca6d17af07a4be923..1ed73f04ad60a81dcc029b61edce002e244ef496 100644 --- a/tests/script/general/http/telegraf.sim +++ b/tests/script/general/http/telegraf.sim @@ -16,224 +16,224 @@ print =============== step1 - parse system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/ print $system_content -if $system_content != @{"status":"error","code":1022,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":5022,"desc":"database name can not be null"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_admin' -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/ print $system_content -if $system_content != @{"status":"error","code":1022,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":5022,"desc":"database name can not be null"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_admin' -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/d123456789012345678901234567890123456 print $system_content -if $system_content != @{"status":"error","code":1023,"desc":"database name too long"}@ then +if $system_content != @{"status":"error","code":5023,"desc":"database name too long"}@ then return -1 endi system_content curl -u root:taosdata -d '[]' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{}]' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"metrics": []}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1025,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":5025,"desc":"metrics size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"metrics": [{}]}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"metrics": 12}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1025,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":5025,"desc":"metrics size is 0"}@ then return -1 endi #system_content curl -u root:taosdata -d '{"metrics": [{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}]}' 127.0.0.1:6020/telegraf/db/root/taosdata1 #print $system_content -#if $system_content != @{"status":"error","code":1026,"desc":"metrics size can not more than 50"}@ then +#if $system_content != @{"status":"error","code":5026,"desc":"metrics size can not more than 50"}@ then # return -1 #endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1027,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5027,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":111,"tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1028,"desc":"metric name type should be string"}@ then +if $system_content != @{"status":"error","code":5028,"desc":"metric name type should be string"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1029,"desc":"metric name length is 0"}@ then +if $system_content != @{"status":"error","code":5029,"desc":"metric name length is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1030,"desc":"metric name length too long"}@ then +if $system_content != @{"status":"error","code":5030,"desc":"metric name length too long"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"}}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1031,"desc":"timestamp not find"}@ then +if $system_content != @{"status":"error","code":5031,"desc":"timestamp not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":""}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1032,"desc":"timestamp type should be integer"}@ then +if $system_content != @{"status":"error","code":5032,"desc":"timestamp type should be integer"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":-1}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1033,"desc":"timestamp value smaller than 0"}@ then +if $system_content != @{"status":"error","code":5033,"desc":"timestamp value smaller than 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1034,"desc":"tags not find"}@ then +if $system_content != @{"status":"error","code":5034,"desc":"tags not find"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1035,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":5035,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":"","timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1035,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":5035,"desc":"tags size is 0"}@ then return -1 endi #system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor","host":"windows","instance":"1","objectname":"Processor","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata #print $system_content -#if $system_content != @{"status":"error","code":1036,"desc":"tags size too long"}@ then +#if $system_content != @{"status":"error","code":5036,"desc":"tags size too long"}@ then # return -1 #endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1035,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":5035,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"":"windows"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1038,"desc":"tag name is null"}@ then +if $system_content != @{"status":"error","code":5038,"desc":"tag name is null"}@ then return -1 endi #system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host111111111111222222222222222222222":""},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 #print $system_content -#if $system_content != @{"status":"error","code":1039,"desc":"tag name length too long"}@ then +#if $system_content != @{"status":"error","code":5039,"desc":"tag name length too long"}@ then # return -1 #endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":true},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1040,"desc":"tag value type should be number or string"}@ then +if $system_content != @{"status":"error","code":5040,"desc":"tag value type should be number or string"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":""},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1041,"desc":"tag value is null"}@ then +if $system_content != @{"status":"error","code":5041,"desc":"tag value is null"}@ then return -1 endi -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"1022":"111"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 +system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"5022":"111"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1042,"desc":"table is null"}@ then +if $system_content != @{"status":"error","code":5042,"desc":"table is null"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1043,"desc":"table name length too long"}@ then +if $system_content != @{"status":"error","code":5043,"desc":"table name length too long"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1045,"desc":"fields size is 0"}@ then +if $system_content != @{"status":"error","code":5045,"desc":"fields size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1048,"desc":"field name is null"}@ then +if $system_content != @{"status":"error","code":5048,"desc":"field name is null"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":"","Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1051,"desc":"field value is null"}@ then +if $system_content != @{"status":"error","code":5051,"desc":"field value is null"}@ then return -1 endi system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":true,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content -if $system_content != @{"status":"error","code":1050,"desc":"field value type should be number or string"}@ then +if $system_content != @{"status":"error","code":5050,"desc":"field value type should be number or string"}@ then return -1 endi diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim index f4ffb5ea9e6655319e5fa9d0a7b0e31133497405..ff1f9f53559c6da89ed2cdaf77c29f1859578ae6 100644 --- a/tests/script/regressionSuite.sim +++ b/tests/script/regressionSuite.sim @@ -124,7 +124,7 @@ run general/parser/slimit.sim run general/parser/fill.sim run general/parser/fill_stb.sim run general/parser/interp.sim -# run general/parser/where.sim +run general/parser/where.sim run general/parser/join.sim run general/parser/join_multivnode.sim run general/parser/select_with_tags.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index ee7093a2ca342059510e95870f83fc35850cd2b2..98b402f4baab49480e012b3a2e636f0f5c1eeacc 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -113,11 +113,11 @@ echo "logDir $LOG_DIR" >> $TAOS_CFG echo "mDebugFlag 135" >> $TAOS_CFG echo "sdbDebugFlag 135" >> $TAOS_CFG echo "dDebugFlag 135" >> $TAOS_CFG -echo "vDebugFlag 143" >> $TAOS_CFG +echo "vDebugFlag 135" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG echo "jnidebugFlag 135" >> $TAOS_CFG echo "odbcdebugFlag 135" >> $TAOS_CFG -echo "httpDebugFlag 135" >> $TAOS_CFG +echo "httpDebugFlag 143" >> $TAOS_CFG echo "monitorDebugFlag 131" >> $TAOS_CFG echo "mqttDebugFlag 131" >> $TAOS_CFG echo "qdebugFlag 135" >> $TAOS_CFG diff --git a/tests/script/unique/http/admin.sim b/tests/script/unique/http/admin.sim index 3a480b6ebed9c8021989f98af385780d54dbf946..10fad091b53e2012e42275e8b99734f2ad2c502e 100644 --- a/tests/script/unique/http/admin.sim +++ b/tests/script/unique/http/admin.sim @@ -33,49 +33,49 @@ print =============== step1 - login system_content curl 127.0.0.1:6020/admin/ print 1-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/admin/xx print 2-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/admin/login print 3-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/admin/login/root print 4-> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi system_content curl 127.0.0.1:6020/admin/login/root/123 print 5-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi system_content curl 127.0.0.1:6020/admin/login/root/123/1/1/3 print 6-> $system_content -if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then +if $system_content != @{"status":"error","code":3,"desc":"auth failure"}@ then return -1 endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.' -d 'show databases' 127.0.0.1:6020/admin/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":1010,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":5010,"desc":"invalid type of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: Taosd eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' 127.0.0.1:6020/admin/login/root/1 print 8-> $system_content -if $system_content != @{"status":"error","code":1053,"desc":"parse http auth token error"}@ then +if $system_content != @{"status":"error","code":5053,"desc":"parse http auth token error"}@ then return -1 endi @@ -105,7 +105,7 @@ endi system_content curl 127.0.0.1:6020/admin/logout print 11 -----> $system_content -if $system_content != @{"status":"error","code":1011,"desc":"no auth info input"}@ then +if $system_content != @{"status":"error","code":5011,"desc":"no auth info input"}@ then return -1 endi @@ -168,7 +168,7 @@ print =============== step7 - use dbs system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'use d1;' 127.0.0.1:6020/admin/all print 23-> $system_content -if $system_content != @{"status":"error","code":1017,"desc":"no need to execute use db cmd"}@ then +if $system_content != @{"status":"error","code":5017,"desc":"no need to execute use db cmd"}@ then return -1 endi diff --git a/tests/script/unique/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim index 2254303e9e29f5de9c91741134af4691d60d5c99..7e2400bb156b86019ad7af600875d8532440a64c 100644 --- a/tests/script/unique/http/opentsdb.sim +++ b/tests/script/unique/http/opentsdb.sim @@ -3,7 +3,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/exec.sh -n dnode1 -s start sleep 3000 @@ -14,92 +13,92 @@ print ============================ dnode1 start print =============== step1 - parse system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/ print $system_content -if $system_content != @{"status":"error","code":1057,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":5057,"desc":"database name can not be null"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db123456789012345678901234567890db print $system_content -if $system_content != @{"status":"error","code":1058,"desc":"database name too long"}@ then +if $system_content != @{"status":"error","code":5058,"desc":"database name too long"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/ print $system_content -if $system_content != @{"status":"error","code":1057,"desc":"database name can not be null"}@ then +if $system_content != @{"status":"error","code":5057,"desc":"database name can not be null"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put2 print $system_content -if $system_content != @{"status":"error","code":1009,"desc":"http url parse error"}@ then +if $system_content != @{"status":"error","code":5009,"desc":"http url parse error"}@ then return -1 endi system_content curl -u root:taosdata -d '[]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1060,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":5060,"desc":"metrics size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1059,"desc":"invalid opentsdb json fromat"}@ then +if $system_content != @{"status":"error","code":5059,"desc":"invalid opentsdb json fromat"}@ then return -1 endi system_content curl -u root:taosdata -d '{}' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1060,"desc":"metrics size is 0"}@ then +if $system_content != @{"status":"error","code":5060,"desc":"metrics size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1062,"desc":"metric name not find"}@ then +if $system_content != @{"status":"error","code":5062,"desc":"metric name not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": 1,"timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1063,"desc":"metric name type should be string"}@ then +if $system_content != @{"status":"error","code":5063,"desc":"metric name type should be string"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1064,"desc":"metric name length is 0"}@ then +if $system_content != @{"status":"error","code":5064,"desc":"metric name length is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "ab1234567890123456789012345678ab1234567890123456789012345678","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":-2147482101,"desc":"tsdb timestamp is out of range"}}],"failed":1,"success":0,"affected_rows":0}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":1547,"desc":"tsdb timestamp is out of range"}}],"failed":1,"success":0,"affected_rows":0}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1066,"desc":"timestamp not find"}@ then +if $system_content != @{"status":"error","code":5066,"desc":"timestamp not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": "2","value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1067,"desc":"timestamp type should be integer"}@ then +if $system_content != @{"status":"error","code":5067,"desc":"timestamp type should be integer"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": -1,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1068,"desc":"timestamp value smaller than 0"}@ then +if $system_content != @{"status":"error","code":5068,"desc":"timestamp value smaller than 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1078,"desc":"value not find"}@ then +if $system_content != @{"status":"error","code":5078,"desc":"value not find"}@ then return -1 endi @@ -107,49 +106,49 @@ endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1069,"desc":"tags not find"}@ then +if $system_content != @{"status":"error","code":5069,"desc":"tags not find"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1070,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":5070,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": 0}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1070,"desc":"tags size is 0"}@ then +if $system_content != @{"status":"error","code":5070,"desc":"tags size is 0"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","group1": "1","group1": "1","group1": "1","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":-2147482782,"desc":"failed to create table"}}],"failed":1,"success":0,"affected_rows":0}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":866,"desc":"failed to create table"}}],"failed":1,"success":0,"affected_rows":0}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"": "web01"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1073,"desc":"tag name is null"}@ then +if $system_content != @{"status":"error","code":5073,"desc":"tag name is null"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host01123456789001123456789001123456789001123456789001123456789001123456789": "01"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1074,"desc":"tag name length too long"}@ then +if $system_content != @{"status":"error","code":5074,"desc":"tag name length too long"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web011234567890011234567890011234567890011234567890011234567890011234567890011234567890011234567890"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1077,"desc":"tag value can not more than 64"}@ then +if $system_content != @{"status":"error","code":5077,"desc":"tag value can not more than 64"}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": ""}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1076,"desc":"tag value is null"}@ then +if $system_content != @{"status":"error","code":5076,"desc":"tag value is null"}@ then return -1 endi @@ -175,11 +174,11 @@ if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09- endi print =============== step3 - multi-query data -system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846401000,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}},{"metric": "sys_cpu","timestamp": 1346846402000,"value": 18,"tags": {"host": "web02","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put +system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846405000,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}},{"metric": "sys_cpu","timestamp": 1346846402000,"value": 18,"tags": {"host": "web02","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbb","table":"sys_cpu_d_bbb_lga_1_web01","timestamp":1346846401000,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"affected_rows":1,"status":"succ"}},{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbb","table":"sys_cpu_d_bbb_lga_1_web02","timestamp":1346846402000,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web02"},"affected_rows":1,"status":"succ"}}],"failed":0,"success":2,"affected_rows":2}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbb","table":"sys_cpu_d_bbb_lga_1_web01","timestamp":1346846405000,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"affected_rows":1,"status":"succ"}},{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbb","table":"sys_cpu_d_bbb_lga_1_web02","timestamp":1346846402000,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web02"},"affected_rows":1,"status":"succ"}}],"failed":0,"success":2,"affected_rows":2}@ then return -1 endi @@ -187,7 +186,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_cpu_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:01.000",18.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:05.000",18.000000000]],"rows":2}@ then return -1 endi @@ -200,7 +199,7 @@ if $system_content != @{"status":"succ","head":["count(*)"],"data":[[3]],"rows": endi print =============== step4 - summary-put data -system_content curl -u root:taosdata -d '[{"metric": "sys_mem","timestamp": 1346846400000,"value": 8,"tags": {"host": "web01","group1": "1","dc": "lga"}},{"metric": "sys_mem","timestamp": 1346846401000,"value": 9,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put?details=false +system_content curl -u root:taosdata -d '[{"metric": "sys_mem","timestamp": 1346846400000,"value": 8,"tags": {"host": "web01","group1": "1","dc": "lga"}},{"metric": "sys_mem","timestamp": 1346846405000,"value": 9,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put?details=false print $system_content @@ -212,7 +211,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_mem_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:01.000",9.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:05.000",9.000000000]],"rows":2}@ then return -1 endi @@ -234,7 +233,7 @@ system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 134 system_content curl -u root:taosdata -d 'select count(*) from db.sys_cpu_d_bbb' 127.0.0.1:6020/rest/sql/ print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[8]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"data":[[7]],"rows":1}@ then return -1 endi diff --git a/tests/stress/.gitignore b/tests/stress/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..25a8031c43c318e14e42873dcceca728aa5a48e1 --- /dev/null +++ b/tests/stress/.gitignore @@ -0,0 +1,3 @@ +stress +stress.exe +cases.json \ No newline at end of file diff --git a/tests/stress/README.md b/tests/stress/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a7f8a2dac6a8500bbbdb177ecbce89ea2b7f97a6 --- /dev/null +++ b/tests/stress/README.md @@ -0,0 +1,80 @@ +# STRESS + +Stress test tool for TDengine. It run a set of test cases randomly and show statistics. + +## COMMAND LINE + +``` bash +$ ./stress [-h=] [-P=<0>] [-d=] [-u=] [-p=] [-c=<4>] [-f=] [-l=] [path_or_sql] +``` + +* **-h**: host name or IP address of TDengine server (default: localhost). +* **-P**: port number of TDengine server (default: 0). +* **-u**: user name (default: root). +* **-p**: password (default: taosdata). +* **-c**: concurrency, number of concurrent goroutines for query (default: 4). +* **-f**: fetch data or not (default: true). +* **-l**: log file path (default: no log). +* **path_or_sql**: a SQL statement or path of a JSON file which contains the test cases (default: cases.json). + +## TEST CASE FILE + +```json +[{ + "weight": 1, + "sql": "select * from meters where ts>=now+%dm and ts<=now-%dm and c1=%v and c2=%d and c3='%s' and tbname='%s'", + "args": [{ + "type": "range", + "min": 30, + "max": 60 + }, { + "type": "bool" + }, { + "type": "int", + "min": -10, + "max": 20 + }, { + "type": "string", + "min": 0, + "max": 10, + }, { + "type": "list", + "list": [ + "table1", + "table2", + "table3", + "table4" + ] + }] +}] +``` + +The test case file is a standard JSON file which contains an array of test cases. For test cases, field `sql` is mandatory, and it can optionally include a `weight` field and an `args` field which is an array of arguments. + +`sql` is a SQL statement, it can include zero or more arguments (placeholders). + +`weight` defines the possibility of the case being selected, the greater value the higher possibility. It must be an non-negative integer and the default value is zero, but, if all cases have a zero weight, all the weights are regarded as 1. + +Placeholders of `sql` are replaced by arguments in `args` at runtime. There are 5 types of arguments currently: + +* **bool**: generate a `boolean` value randomly. +* **int**: generate an `integer` between [`min`, `max`] randomly, the default value of `min` is 0 and `max` is 100. +* **range**: generate two `integer`s between [`min`, `max`] randomly, the first is less than the second, the default value of `min` is 0 and `max` is 100. +* **string**: generate a `string` with length between [`min`, `max`] randomly, the default value of `min` is 0 and `max` is 100. +* **list**: select an item from `list` randomly. + +## OUTPUT + +``` + 00:00:08 | TOTAL REQ | TOTAL TIME(us) | TOTAL AVG(us) | REQUEST | TIME(us) | AVERAGE(us) | + TOTAL | 3027 | 26183890 | 8650.11 | 287 | 3060935 | 10665.28 | + SUCCESS | 3027 | 26183890 | 8650.11 | 287 | 3060935 | 10665.28 | + FAIL | 0 | 0 | 0.00 | 0 | 0 | 0.00 | +``` + +* **Col 2**: total number of request since test start. +* **Col 3**: total time of all request since test start. +* **Col 4**: average time of all request since test start. +* **Col 5**: number of request in last second. +* **Col 6**: time of all request in last second. +* **Col 7**: average time of all request in last second. diff --git a/tests/stress/go.mod b/tests/stress/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..df9b2806b551429ee59ad0d44e3eb0ad75ab39cb --- /dev/null +++ b/tests/stress/go.mod @@ -0,0 +1,7 @@ +module github.com/taosdata/stress + +go 1.14 + +require ( + github.com/taosdata/driver-go v0.0.0-20200606095205-b786bac1857f +) diff --git a/tests/stress/main.go b/tests/stress/main.go new file mode 100644 index 0000000000000000000000000000000000000000..c3b9290a379b0782f1042909cc0cadb728a4620e --- /dev/null +++ b/tests/stress/main.go @@ -0,0 +1,406 @@ +package main + +import ( + "database/sql" + "encoding/json" + "errors" + "flag" + "fmt" + "math/rand" + "os" + "os/signal" + "strings" + "sync" + "sync/atomic" + "time" + + _ "github.com/taosdata/driver-go/taosSql" +) + +type argument struct { + Type string `json:"type"` + Min int `json:"min"` + Max int `json:"max"` + List []interface{} `json:"list, omitempty"` +} + +type testCase struct { + isQuery bool `json:"-"` + numArgs int `json:"-"` + Weight int `json:"weight"` + SQL string `json:"sql"` + Args []argument `json:"args"` +} + +func (arg *argument) check() (int, error) { + if arg.Type == "list" { + if len(arg.List) == 0 { + return 0, errors.New("list cannot be empty") + } + return 1, nil + } + + if arg.Max < arg.Min { + return 0, errors.New("invalid min/max value") + } + + if arg.Type == "string" { + if arg.Min < 0 { + return 0, errors.New("negative string length") + } + } + + if arg.Type == "int" && arg.Min == 0 && arg.Max == 0 { + arg.Max = arg.Min + 100 + } + + if arg.Type == "range" { + return 2, nil + } + + return 1, nil +} + +func (arg *argument) generate(args []interface{}) []interface{} { + const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + + switch arg.Type { + case "bool": + if rand.Intn(2) == 1 { + args = append(args, true) + } else { + args = append(args, false) + } + + case "int": + v := rand.Intn(arg.Max-arg.Min+1) + arg.Min + args = append(args, v) + + case "range": + v := rand.Intn(arg.Max-arg.Min) + arg.Min + args = append(args, v) + v = rand.Intn(arg.Max-v+1) + v + args = append(args, v) + + case "string": + l := rand.Intn(arg.Max-arg.Min+1) + arg.Min + sb := strings.Builder{} + for i := 0; i < l; i++ { + sb.WriteByte(chars[rand.Intn(len(chars))]) + } + args = append(args, sb.String()) + + case "list": + v := arg.List[rand.Intn(len(arg.List))] + args = append(args, v) + } + + return args +} + +func (tc *testCase) buildSql() string { + args := make([]interface{}, 0, tc.numArgs) + for i := 0; i < len(tc.Args); i++ { + args = tc.Args[i].generate(args) + } + return fmt.Sprintf(tc.SQL, args...) +} + +type statitics struct { + succeeded int64 + failed int64 + succeededDuration int64 + failedDuration int64 +} + +var ( + host string + port uint + database string + user string + password string + fetch bool + + chLog chan string + wgLog sync.WaitGroup + startAt time.Time + shouldStop int64 + wgTest sync.WaitGroup + stat statitics + totalWeight int + cases []testCase +) + +func loadTestCaseFromFile(file *os.File) error { + if e := json.NewDecoder(file).Decode(&cases); e != nil { + return e + } + + if len(cases) == 0 { + return fmt.Errorf("no test case loaded.") + } + + for i := 0; i < len(cases); i++ { + c := &cases[i] + c.SQL = strings.TrimSpace(c.SQL) + c.isQuery = strings.ToLower(c.SQL[:6]) == "select" + if c.Weight < 0 { + return fmt.Errorf("test %d: negative weight", i) + } + totalWeight += c.Weight + + for j := 0; j < len(c.Args); j++ { + arg := &c.Args[j] + arg.Type = strings.ToLower(arg.Type) + n, e := arg.check() + if e != nil { + return fmt.Errorf("test case %d argument %d: %s", i, j, e.Error()) + } + c.numArgs += n + } + } + + if totalWeight == 0 { + for i := 0; i < len(cases); i++ { + cases[i].Weight = 1 + } + totalWeight = len(cases) + } + + return nil +} + +func loadTestCase(pathOrSQL string) error { + if f, e := os.Open(pathOrSQL); e == nil { + defer f.Close() + return loadTestCaseFromFile(f) + } + + pathOrSQL = strings.TrimSpace(pathOrSQL) + if strings.ToLower(pathOrSQL[:6]) != "select" { + return fmt.Errorf("'%s' is not a valid file or SQL statement", pathOrSQL) + } + + cases = append(cases, testCase{ + isQuery: true, + Weight: 1, + numArgs: 0, + SQL: pathOrSQL, + }) + totalWeight = 1 + + return nil +} + +func selectTestCase() *testCase { + sum, target := 0, rand.Intn(totalWeight) + var c *testCase + for i := 0; i < len(cases); i++ { + c = &cases[i] + sum += c.Weight + if sum > target { + break + } + } + return c +} + +func runTest() { + defer wgTest.Done() + db, e := sql.Open("taosSql", fmt.Sprintf("%s:%s@tcp(%s:%v)/%s", user, password, host, port, database)) + if e != nil { + fmt.Printf("failed to connect to database: %s\n", e.Error()) + return + } + defer db.Close() + + for atomic.LoadInt64(&shouldStop) == 0 { + c := selectTestCase() + str := c.buildSql() + + start := time.Now() + if c.isQuery { + var rows *sql.Rows + if rows, e = db.Query(str); rows != nil { + if fetch { + for rows.Next() { + } + } + rows.Close() + } + } else { + _, e = db.Exec(str) + } + duration := time.Now().Sub(start).Microseconds() + + if e != nil { + if chLog != nil { + chLog <- str + ": " + e.Error() + } + atomic.AddInt64(&stat.failed, 1) + atomic.AddInt64(&stat.failedDuration, duration) + } else { + atomic.AddInt64(&stat.succeeded, 1) + atomic.AddInt64(&stat.succeededDuration, duration) + } + } +} + +func getStatPrinter() func(tm time.Time) { + var last statitics + lastPrintAt := startAt + + return func(tm time.Time) { + var current statitics + + current.succeeded = atomic.LoadInt64(&stat.succeeded) + current.failed = atomic.LoadInt64(&stat.failed) + current.succeededDuration = atomic.LoadInt64(&stat.succeededDuration) + current.failedDuration = atomic.LoadInt64(&stat.failedDuration) + + seconds := int64(tm.Sub(startAt).Seconds()) + format := "\033[47;30m %02v:%02v:%02v | TOTAL REQ | TOTAL TIME(us) | TOTAL AVG(us) | REQUEST | TIME(us) | AVERAGE(us) |\033[0m\n" + fmt.Printf(format, seconds/3600, seconds%3600/60, seconds%60) + + tr := current.succeeded + current.failed + td := current.succeededDuration + current.failedDuration + r := tr - last.succeeded - last.failed + d := td - last.succeededDuration - last.failedDuration + ta, a := 0.0, 0.0 + if tr > 0 { + ta = float64(td) / float64(tr) + } + if r > 0 { + a = float64(d) / float64(r) + } + format = " TOTAL | %9v | %14v | %13.2f | %7v | %10v | % 13.2f |\n" + fmt.Printf(format, tr, td, ta, r, d, a) + + tr = current.succeeded + td = current.succeededDuration + r = tr - last.succeeded + d = td - last.succeededDuration + ta, a = 0.0, 0.0 + if tr > 0 { + ta = float64(td) / float64(tr) + } + if r > 0 { + a = float64(d) / float64(r) + } + format = " SUCCESS | \033[32m%9v\033[0m | \033[32m%14v\033[0m | \033[32m%13.2f\033[0m | \033[32m%7v\033[0m | \033[32m%10v\033[0m | \033[32m%13.2f\033[0m |\n" + fmt.Printf(format, tr, td, ta, r, d, a) + + tr = current.failed + td = current.failedDuration + r = tr - last.failed + d = td - last.failedDuration + ta, a = 0.0, 0.0 + if tr > 0 { + ta = float64(td) / float64(tr) + } + if r > 0 { + a = float64(d) / float64(r) + } + format = " FAIL | \033[31m%9v\033[0m | \033[31m%14v\033[0m | \033[31m%13.2f\033[0m | \033[31m%7v\033[0m | \033[31m%10v\033[0m | \033[31m%13.2f\033[0m |\n" + fmt.Printf(format, tr, td, ta, r, d, a) + + last = current + lastPrintAt = tm + } +} + +func startLogger(path string) error { + if len(path) == 0 { + return nil + } + + f, e := os.Create(path) + if e != nil { + return e + } + + chLog = make(chan string, 100) + wgLog.Add(1) + go func() { + for s := range chLog { + if f != nil { + f.WriteString(s) + f.WriteString("\n") + } + } + f.Close() + wgLog.Done() + }() + + return nil +} + +func main() { + var concurrency uint + var logPath string + flag.StringVar(&host, "h", "localhost", "host name or IP address of TDengine server") + flag.UintVar(&port, "P", 0, "port (default 0)") + flag.StringVar(&database, "d", "test", "database name") + flag.StringVar(&user, "u", "root", "user name") + flag.StringVar(&password, "p", "taosdata", "password") + flag.BoolVar(&fetch, "f", true, "fetch result or not") + flag.UintVar(&concurrency, "c", 4, "concurrency, number of goroutines for query") + flag.StringVar(&logPath, "l", "", "path of log file (default: no log)") + flag.Parse() + + if e := startLogger(logPath); e != nil { + fmt.Println("failed to open log file:", e.Error()) + return + } + + pathOrSQL := flag.Arg(0) + if len(pathOrSQL) == 0 { + pathOrSQL = "cases.json" + } + if e := loadTestCase(pathOrSQL); e != nil { + fmt.Println("failed to load test cases:", e.Error()) + return + } + + rand.Seed(time.Now().UnixNano()) + + fmt.Printf("\nSERVER: %s DATABASE: %s CONCURRENCY: %d FETCH DATA: %v\n\n", host, database, concurrency, fetch) + + startAt = time.Now() + printStat := getStatPrinter() + printStat(startAt) + + for i := uint(0); i < concurrency; i++ { + wgTest.Add(1) + go runTest() + } + + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt) + ticker := time.NewTicker(time.Second) + + fmt.Println("Ctrl + C to exit....\033[1A") + +LOOP: + for { + select { + case <-interrupt: + break LOOP + case tm := <-ticker.C: + fmt.Print("\033[4A") + printStat(tm) + } + } + + atomic.StoreInt64(&shouldStop, 1) + fmt.Print("\033[100D'Ctrl + C' received, Waiting started query to stop...") + wgTest.Wait() + + if chLog != nil { + close(chLog) + wgLog.Wait() + } + fmt.Print("\033[4A\033[100D") + printStat(time.Now()) + fmt.Println() +}