diff --git a/example/src/tmq.c b/example/src/tmq.c index 094fd94bfc00a137073dae63a41e4754fb922ced..35c3e655d61e75fb0d7ea8e162e6fdcde7b6009b 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -160,7 +160,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); + tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1000); if (tmqmessage) { msg_process(tmqmessage); tmq_message_destroy(tmqmessage); diff --git a/include/client/taos.h b/include/client/taos.h index 2c8135c8ffaf5e920e76544f5a0796df6d5d0eda..8b1517c6fffc7613b9366d2f0943520243cba5d7 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -224,10 +224,8 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); /* ------------------------TMQ CONSUMER INTERFACE------------------------ */ DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); -#if 0 -DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); -DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics); -#endif +DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); +DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); #if 0 diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 37d20cdb974b8f56b8f69b8ecba64815d882c7f7..a04e2afc947abc56fff1c9b669f7dac1346417e8 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -49,6 +49,11 @@ enum { TMQ_CONF__RESET_OFFSET__NONE = -3, }; +enum { + TMQ_MSG_TYPE__POLL_RSP = 0, + TMQ_MSG_TYPE__EP_RSP, +}; + typedef struct { uint32_t numOfTables; SArray* pGroupList; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a739e51d6b2688020e9a81cad47d487a3e6e00fc..ab0472a575d9f1fbcd0ca4542564aaafa790417d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -167,6 +167,7 @@ typedef struct { typedef struct { char db[TSDB_DB_FNAME_LEN]; + int64_t dbId; int32_t vgVersion; } SBuildUseDBInput; @@ -563,6 +564,7 @@ int32_t tDeserializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; + int64_t dbId; int32_t vgVersion; } SUseDbReq; @@ -1380,8 +1382,6 @@ typedef struct SMqCMGetSubEpReq { char cgroup[TSDB_CONSUMER_GROUP_LEN]; } SMqCMGetSubEpReq; -#pragma pack(pop) - static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pMsg->contLen); @@ -1851,6 +1851,12 @@ typedef struct { SMqTbData* tbData; } SMqTopicData; +typedef struct { + int8_t mqMsgType; + int32_t code; + int32_t epoch; +} SMqRspHead; + typedef struct { int64_t consumerId; SSchemaWrapper* schemas; @@ -1867,6 +1873,7 @@ typedef struct { int64_t consumerId; int64_t blockingTime; + int32_t epoch; char cgroup[TSDB_CONSUMER_GROUP_LEN]; int64_t currentOffset; @@ -1886,11 +1893,19 @@ typedef struct { typedef struct { int64_t consumerId; - int32_t epoch; char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray* topics; // SArray } SMqCMGetSubEpRsp; +struct tmq_message_t { + SMqRspHead head; + union { + SMqConsumeRsp consumeRsp; + SMqCMGetSubEpRsp getEpRsp; + }; + void* extra; +}; + static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { @@ -1943,7 +1958,6 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); - tlen += taosEncodeFixedI32(buf, pRsp->epoch); tlen += taosEncodeString(buf, pRsp->cgroup); int32_t sz = taosArrayGetSize(pRsp->topics); tlen += taosEncodeFixedI32(buf, sz); @@ -1956,7 +1970,6 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { buf = taosDecodeFixedI64(buf, &pRsp->consumerId); - buf = taosDecodeFixedI32(buf, &pRsp->epoch); buf = taosDecodeStringTo(buf, pRsp->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); @@ -1972,6 +1985,8 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p return buf; } +#pragma pack(pop) + #ifdef __cplusplus } #endif diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index ebe7c92d31b26779f4f478c332acafc862ba71a7..c6183780f906606ea0542f2a367093dc0ad860dc 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -95,7 +95,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); */ void catalogFreeHandle(SCatalog* pCatalog); -int32_t catalogGetDBVgVersion(SCatalog* pCatalog, const char* dbName, int32_t* version); +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId); /** * Get a DB's all vgroup info. diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 8009f5dc11ed073be13b62354263c2ed7a63ac75..eadf618d087bac94000f0c5256a08b646bdafc86 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -118,14 +118,16 @@ typedef struct SNodeList { SListCell* pTail; } SNodeList; -SNode* nodesMakeNode(ENodeType type); -void nodesDestroyNode(SNode* pNode); +#define SNodeptr void* + +SNodeptr nodesMakeNode(ENodeType type); +void nodesDestroyNode(SNodeptr pNode); SNodeList* nodesMakeList(); -int32_t nodesListAppend(SNodeList* pList, SNode* pNode); +int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); -SNode* nodesListGetNode(SNodeList* pList, int32_t index); +SNodeptr nodesListGetNode(SNodeList* pList, int32_t index); void nodesDestroyList(SNodeList* pList); typedef enum EDealRes { @@ -135,9 +137,9 @@ typedef enum EDealRes { } EDealRes; typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext); -void nodesWalkNode(SNode* pNode, FNodeWalker walker, void* pContext); +void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext); void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext); -void nodesWalkNodePostOrder(SNode* pNode, FNodeWalker walker, void* pContext); +void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext); void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext); @@ -146,12 +148,12 @@ void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext); void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); -bool nodesEqualNode(const SNode* a, const SNode* b); +bool nodesEqualNode(const SNodeptr a, const SNodeptr b); -SNode* nodesCloneNode(const SNode* pNode); +SNodeptr nodesCloneNode(const SNodeptr pNode); SNodeList* nodesCloneList(const SNodeList* pList); -int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen); +int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToNode(const char* pStr, SNode** pNode); #ifdef __cplusplus diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 76f23da1e3f9763f76cf7fa0c94e03a62bc5596a..6d3f97fc4e3e0d0f335af3cc269713309f739e35 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -156,6 +156,8 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); */ int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); +int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp); + void initQueryModuleMsgHandle(); const SSchema* tGetTbnameColumnSchema(); diff --git a/include/os/osDir.h b/include/os/osDir.h index e328c420638eb1d146abe97f8abf6bacb6286377..eda83ea0ea85a5cf4e03cdf620c357c34294ec0c 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -26,6 +26,7 @@ int32_t taosMkDir(const char *dirname); void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen); +bool taosIsDir(const char *dirname); #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index a0771dc7340562787a0e8bd69fbb880998fb8dcc..35ac032a8a65d4d530893478d7051cee2bd5565b 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -33,21 +33,26 @@ typedef struct { SDiskSize size; } SDiskSpace; -int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); -int32_t taosGetCpuCores(); void taosGetSystemInfo(); +bool taosGetEmail(char *email, int32_t maxLen); +bool taosGetOsReleaseName(char *releaseName, int32_t maxLen); +bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores); +int32_t taosGetCpuCores(); +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); +bool taosGetTotalSysMemoryKB(uint64_t *kb); +bool taosGetProcMemory(float *memoryUsedMB); // +bool taosGetSysMemory(float *memoryUsedMB); // +void taosGetDisk(); +int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); bool taosReadProcIO(int64_t *rchars, int64_t *wchars); bool taosGetProcIO(float *readKB, float *writeKB); bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes); bool taosGetBandSpeed(float *bandSpeedKb); -void taosGetDisk(); -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); -bool taosGetProcMemory(float *memoryUsedMB); -bool taosGetSysMemory(float *memoryUsedMB); -int taosSystem(const char *cmd); + +int32_t taosSystem(const char *cmd); void taosKillSystem(); int32_t taosGetSystemUUID(char *uid, int32_t uidlen); -char * taosGetCmdlineByPID(int pid); +char *taosGetCmdlineByPID(int32_t pid); void taosSetCoreDump(bool enable); typedef struct { diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 8275054a64f188c6f30a70a481f9ff9f9ab95d2b..0e1d352f9a8b2cdc4ab9bffb728868738b8b9cac 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -82,7 +82,7 @@ typedef struct SConfig { SConfig *cfgInit(); int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr); -int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs); // SConfigPair +int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); diff --git a/include/util/thttp.h b/include/util/thttp.h new file mode 100644 index 0000000000000000000000000000000000000000..f211b2615d8d166ad82bf0125655a7efdb58ff94 --- /dev/null +++ b/include/util/thttp.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_HTTP_H_ +#define _TD_UTIL_HTTP_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, int32_t contLen); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_UTIL_H_*/ diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 45c1858948e1377afb2e9f83a859e4ccc2562283..e9e64a78b8f57e94ec81515a494bbf2b10970a94 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -13,19 +13,17 @@ * along with this program. If not, see . */ -#include "clientInt.h" -#include "trpc.h" #include "catalog.h" +#include "clientInt.h" #include "clientLog.h" +#include "trpc.h" static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); -static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) { - return 0; -} +static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; @@ -39,8 +37,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); - tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%"PRIx64, rsp->db, rsp->vgVersion, rsp->uid); - + tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->uid); + if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); } else { @@ -60,8 +58,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog taosHashCleanup(vgInfo.vgHash); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - } - + } + catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, &vgInfo); } @@ -106,8 +104,8 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo return TSDB_CODE_SUCCESS; } -static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) { - SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); +static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { + SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); if (NULL == info) { tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType); return TSDB_CODE_SUCCESS; @@ -116,7 +114,7 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRs int32_t kvNum = pRsp->info ? taosArrayGetSize(pRsp->info) : 0; tscDebug("hb got %d rsp kv", kvNum); - + for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pRsp->info, i); switch (kv->key) { @@ -126,30 +124,30 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRs break; } - int64_t *clusterId = (int64_t *)info->param; + int64_t *clusterId = (int64_t *)info->param; struct SCatalog *pCatalog = NULL; - + int32_t code = catalogGetHandle(*clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); break; } hbProcessDBInfoRsp(kv->value, kv->valueLen, pCatalog); break; } - case HEARTBEAT_KEY_STBINFO:{ + case HEARTBEAT_KEY_STBINFO: { if (kv->valueLen <= 0 || NULL == kv->value) { tscError("invalid hb stb info, len:%d, value:%p", kv->valueLen, kv->value); break; } - int64_t *clusterId = (int64_t *)info->param; + int64_t *clusterId = (int64_t *)info->param; struct SCatalog *pCatalog = NULL; - + int32_t code = catalogGetHandle(*clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); break; } @@ -165,22 +163,22 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRs return TSDB_CODE_SUCCESS; } -static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code) { +static int32_t hbMqAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) { static int32_t emptyRspNum = 0; if (code != 0) { tfree(param); return -1; } - char *key = (char *)param; + char *key = (char *)param; SClientHbBatchRsp pRsp = {0}; tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); - + int32_t rspNum = taosArrayGetSize(pRsp.rsps); - SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL || NULL == *pInst) { - tscError("cluster not exist, key:%s", key); + tscError("cluster not exist, key:%s", key); tfree(param); tFreeClientHbBatchRsp(&pRsp); return -1; @@ -189,13 +187,14 @@ static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code tfree(param); if (rspNum) { - tscDebug("hb got %d rsp, %d empty rsp received before", rspNum, atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0)); + tscDebug("hb got %d rsp, %d empty rsp received before", rspNum, + atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0)); } else { atomic_add_fetch_32(&emptyRspNum, 1); } for (int32_t i = 0; i < rspNum; ++i) { - SClientHbRsp* rsp = taosArrayGet(pRsp.rsps, i); + SClientHbRsp *rsp = taosArrayGet(pRsp.rsps, i); code = (*clientHbMgr.rspHandle[rsp->connKey.hbType])((*pInst)->pAppHbMgr, rsp); if (code) { break; @@ -203,14 +202,14 @@ static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code } tFreeClientHbBatchRsp(&pRsp); - + return code; } int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SDbVgVersion *dbs = NULL; - uint32_t dbNum = 0; - int32_t code = 0; + uint32_t dbNum = 0; + int32_t code = 0; code = catalogGetExpiredDBs(pCatalog, &dbs, &dbNum); if (TSDB_CODE_SUCCESS != code) { @@ -238,8 +237,8 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SSTableMetaVersion *stbs = NULL; - uint32_t stbNum = 0; - int32_t code = 0; + uint32_t stbNum = 0; + int32_t code = 0; code = catalogGetExpiredSTables(pCatalog, &stbs, &stbNum); if (TSDB_CODE_SUCCESS != code) { @@ -254,7 +253,7 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC SSTableMetaVersion *stb = &stbs[i]; stb->suid = htobe64(stb->suid); stb->sversion = htons(stb->sversion); - stb->tversion = htons(stb->tversion); + stb->tversion = htons(stb->tversion); } SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs}; @@ -266,17 +265,16 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC return TSDB_CODE_SUCCESS; } - -int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req) { - int64_t *clusterId = (int64_t *)param; +int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { + int64_t *clusterId = (int64_t *)param; struct SCatalog *pCatalog = NULL; int32_t code = catalogGetHandle(*clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); return code; } - + code = hbGetExpiredDBInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { return code; @@ -287,13 +285,10 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req return code; } - return TSDB_CODE_SUCCESS; } -int32_t hbMqHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req) { - -} +int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) {} void hbMgrInitMqHbHandle() { clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle; @@ -312,10 +307,8 @@ void hbFreeReq(void *req) { tFreeReqKvHash(pReq->info); } - - -SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { - SClientHbBatchReq* pBatchReq = calloc(1, sizeof(SClientHbBatchReq)); +SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { + SClientHbBatchReq *pBatchReq = calloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -324,11 +317,11 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); int32_t code = 0; - void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); + void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); while (pIter != NULL) { - SClientHbReq* pOneReq = pIter; + SClientHbReq *pOneReq = pIter; - SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pOneReq->connKey, sizeof(SClientHbKey)); + SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pOneReq->connKey, sizeof(SClientHbKey)); if (info) { code = (*clientHbMgr.reqHandle[pOneReq->connKey.hbType])(&pOneReq->connKey, info->param, pOneReq); if (code) { @@ -350,11 +343,10 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return pBatchReq; } - void hbClearReqInfo(SAppHbMgr *pAppHbMgr) { void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); while (pIter != NULL) { - SClientHbReq* pOneReq = pIter; + SClientHbReq *pOneReq = pIter; tFreeReqKvHash(pOneReq->info); taosHashClear(pOneReq->info); @@ -363,31 +355,29 @@ void hbClearReqInfo(SAppHbMgr *pAppHbMgr) { } } - - -static void* hbThreadFunc(void* param) { +static void *hbThreadFunc(void *param) { setThreadName("hb"); while (1) { int8_t threadStop = atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 1, 2); - if(1 == threadStop) { + if (1 == threadStop) { break; } pthread_mutex_lock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); - for(int i = 0; i < sz; i++) { - SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); + for (int i = 0; i < sz; i++) { + SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); int32_t connCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); if (connCnt == 0) { continue; } - SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr); + SClientHbBatchReq *pReq = hbGatherAllInfo(pAppHbMgr); if (pReq == NULL) { continue; } - int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); + int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); void *buf = malloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -395,7 +385,7 @@ static void* hbThreadFunc(void* param) { hbClearReqInfo(pAppHbMgr); break; } - + tSerializeSClientHbBatchReq(buf, tlen, pReq); SMsgSendInfo *pInfo = calloc(1, sizeof(SMsgSendInfo)); @@ -415,17 +405,17 @@ static void* hbThreadFunc(void* param) { pInfo->requestObjRefId = 0; SAppInstInfo *pAppInstInfo = pAppHbMgr->pAppInstInfo; - int64_t transporterId = 0; - SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); + int64_t transporterId = 0; + SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); - tFreeClientHbBatchReq(pReq, false); + tFreeClientHbBatchReq(pReq, false); hbClearReqInfo(pAppHbMgr); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } pthread_mutex_unlock(&clientHbMgr.lock); - + taosMsleep(HEARTBEAT_INTERVAL); } return NULL; @@ -449,17 +439,18 @@ static void hbStopThread() { tscDebug("hb thread already stopped"); return; } - + while (2 != atomic_load_8(&clientHbMgr.threadStop)) { usleep(10); } - tscDebug("hb thread stopped"); + tscDebug("hb thread stopped"); } -SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { +SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { + /*return NULL;*/ hbMgrInit(); - SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr)); + SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -495,7 +486,7 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { pthread_mutex_lock(&clientHbMgr.lock); taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); pthread_mutex_unlock(&clientHbMgr.lock); - + return pAppHbMgr; } @@ -504,7 +495,7 @@ void appHbMgrCleanup(void) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { - SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); + SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); taosHashCleanup(pTarget->activeInfo); pTarget->activeInfo = NULL; taosHashCleanup(pTarget->connInfo); @@ -515,11 +506,12 @@ void appHbMgrCleanup(void) { } int hbMgrInit() { + /*return 0;*/ // init once int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); if (old == 1) return 0; - clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void*)); + clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); pthread_mutex_init(&clientHbMgr.lock, NULL); // init handle funcs @@ -534,34 +526,34 @@ int hbMgrInit() { void hbMgrCleanUp() { return; hbStopThread(); - + // destroy all appHbMgr int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0); if (old == 0) return; pthread_mutex_lock(&clientHbMgr.lock); appHbMgrCleanup(); - taosArrayDestroy(clientHbMgr.appHbMgrs); + taosArrayDestroy(clientHbMgr.appHbMgrs); pthread_mutex_unlock(&clientHbMgr.lock); - + clientHbMgr.appHbMgrs = NULL; } -int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) { +int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) { // init hash in activeinfo - void* data = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + void *data = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (data != NULL) { return 0; } SClientHbReq hbReq; hbReq.connKey = connKey; hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - + taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq)); - + // init hash if (info != NULL) { - SClientHbReq * pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); info->req = pReq; taosHashPut(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey), info, sizeof(SHbConnInfo)); } @@ -570,9 +562,10 @@ int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo * return 0; } -int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { +int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { + /*return 0;*/ SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY}; - SHbConnInfo info = {0}; + SHbConnInfo info = {0}; switch (hbType) { case HEARTBEAT_TYPE_QUERY: { @@ -588,11 +581,12 @@ int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int3 default: break; } - + return hbRegisterConnImpl(pAppHbMgr, connKey, &info); } -void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) { +void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { + /*return;*/ int32_t code = 0; code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); @@ -602,9 +596,11 @@ void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) { atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } -int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen) { +int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen, + int32_t valueLen) { + return 0; // find req by connection id - SClientHbReq* pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); ASSERT(pReq != NULL); taosHashPut(pReq->info, key, keyLen, value, valueLen); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 454fb7f1fea694ca61cc83af4458bd262a241567..9534c1164631b1788e6ee6447ca6c0b289e523f0 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -19,6 +19,7 @@ #include "clientInt.h" #include "clientLog.h" #include "catalog.h" +#include "query.h" int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); @@ -243,6 +244,23 @@ int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; + if (TSDB_CODE_MND_DB_NOT_EXIST == code) { + SUseDbRsp usedbRsp = {0}; + tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); + struct SCatalog *pCatalog = NULL; + + if (usedbRsp.vgVersion >= 0) { + int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code)); + } else { + catalogRemoveDB(pCatalog, usedbRsp.db, usedbRsp.uid); + } + } + + tFreeSUsedbRsp(&usedbRsp); + } + if (code != TSDB_CODE_SUCCESS) { free(pMsg->pData); setErrno(pRequest, code); @@ -256,6 +274,26 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { SName name = {0}; tNameFromString(&name, usedbRsp.db, T_NAME_ACCT|T_NAME_DB); + SUseDbOutput output = {0}; + code = queryBuildUseDbOutput(&output, &usedbRsp); + + if (code != 0) { + terrno = code; + if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash); + tfree(output.dbVgroup); + + tscError("failed to build use db output since %s", terrstr()); + } else { + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tstrerror(code)); + } else { + catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup); + } + } + tFreeSUsedbRsp(&usedbRsp); char db[TSDB_DB_NAME_LEN] = {0}; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 6271b285465dd567252488cee8975b5c77fb766e..501d8ce571fe8f8e04832548e5b6e4399983fc62 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -25,6 +25,7 @@ #include "tglobal.h" #include "tmsgtype.h" #include "tpagedbuf.h" +#include "tqueue.h" #include "tref.h" struct tmq_list_t { @@ -58,22 +59,28 @@ struct tmq_t { char groupId[256]; char clientId[256]; int8_t autoCommit; - SRWLatch lock; int64_t consumerId; int32_t epoch; int32_t resetOffsetCfg; int64_t status; - tsem_t rspSem; STscObj* pTscObj; tmq_commit_cb* commit_cb; int32_t nextTopicIdx; SArray* clientTopics; // SArray + STaosQueue* mqueue; // queue of tmq_message_t + STaosQall* qall; // stat int64_t pollCnt; }; -struct tmq_message_t { - SMqConsumeRsp rsp; +enum { + TMQ_VG_STATUS__IDLE = 0, + TMQ_VG_STATUS__WAIT, +}; + +enum { + TMQ_CONSUMER_STATUS__INIT = 0, + TMQ_CONSUMER_STATUS__READY, }; typedef struct { @@ -83,6 +90,7 @@ typedef struct { int64_t currentOffset; // connection info int32_t vgId; + int32_t vgStatus; SEpSet epSet; } SMqClientVg; @@ -104,15 +112,16 @@ typedef struct { typedef struct { tmq_t* tmq; - int32_t wait; + int32_t sync; + tsem_t rspSem; } SMqAskEpCbParam; typedef struct { - tmq_t* tmq; - SMqClientVg* pVg; - tmq_message_t** retMsg; - tsem_t rspSem; -} SMqConsumeCbParam; + tmq_t* tmq; + SMqClientVg* pVg; + int32_t epoch; + tsem_t rspSem; +} SMqPollCbParam; typedef struct { tmq_t* tmq; @@ -125,7 +134,7 @@ typedef struct { tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); conf->auto_commit = false; - conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; + conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; return conf; } @@ -209,6 +218,22 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } +tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { + if (*topics == NULL) { + *topics = tmq_list_new(); + } + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* topic = taosArrayGetP(tmq->clientTopics, i); + tmq_list_append(*topics, strdup(topic->topicName)); + } + return TMQ_RESP_ERR__SUCCESS; +} + +tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) { + tmq_list_t* lst = tmq_list_new(); + return tmq_subscribe(tmq, lst); +} + tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = calloc(sizeof(tmq_t), 1); if (pTmq == NULL) { @@ -218,7 +243,6 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->status = 0; pTmq->pollCnt = 0; pTmq->epoch = 0; - taosInitRWLatch(&pTmq->lock); // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -226,9 +250,11 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->commit_cb = conf->commit_cb; pTmq->resetOffsetCfg = conf->resetOffset; - tsem_init(&pTmq->rspSem, 0, 0); pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); + + pTmq->mqueue = taosOpenQueue(); + pTmq->qall = taosAllocateQall(); return pTmq; } @@ -290,7 +316,11 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in pParam->tmq = tmq; tsem_init(&pParam->rspSem, 0, 0); - pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; + pRequest->body.requestMsg = (SDataBuf){ + .pData = buf, + .len = tlen, + .handle = NULL, + }; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->param = pParam; @@ -365,10 +395,17 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tscError("failed to malloc request"); } - SMqSubscribeCbParam param = {.rspErr = TMQ_RESP_ERR__SUCCESS, .tmq = tmq}; + SMqSubscribeCbParam param = { + .rspErr = TMQ_RESP_ERR__SUCCESS, + .tmq = tmq, + }; tsem_init(¶m.rspSem, 0, 0); - pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL}; + pRequest->body.requestMsg = (SDataBuf){ + .pData = buf, + .len = tlen, + .handle = NULL, + }; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->param = ¶m; @@ -391,36 +428,6 @@ _return: void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; } -SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { - tmq_t* pTmq = (void*)param; - SArray* pArray = taosArrayInit(0, sizeof(SKv)); - if (pArray == NULL) { - return NULL; - } - SKv kv = {0}; - kv.key = HEARTBEAT_KEY_MQ_TMP; - - SMqHbMsg* pMqHb = malloc(sizeof(SMqHbMsg)); - if (pMqHb == NULL) { - return pArray; - } - pMqHb->consumerId = connKey.connId; - SArray* clientTopics = pTmq->clientTopics; - int sz = taosArrayGetSize(clientTopics); - for (int i = 0; i < sz; i++) { - SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i); - /*if (pCTopic->vgId == -1) {*/ - /*pMqHb->status = 1;*/ - /*break;*/ - /*}*/ - } - kv.value = pMqHb; - kv.valueLen = sizeof(SMqHbMsg); - taosArrayPush(pArray, &kv); - - return pArray; -} - TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { STscObj* pTscObj = (STscObj*)taos; SRequestObj* pRequest = NULL; @@ -578,7 +585,7 @@ void tmqShowMsg(tmq_message_t* tmq_message) { static bool noPrintSchema; char pBuf[128]; - SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; int32_t colNum = pRsp->schemas->nCols; if (!noPrintSchema) { printf("|"); @@ -618,94 +625,125 @@ void tmqShowMsg(tmq_message_t* tmq_message) { } int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { - SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; - SMqClientVg* pVg = pParam->pVg; + printf("recv poll\n"); + SMqPollCbParam* pParam = (SMqPollCbParam*)param; + SMqClientVg* pVg = pParam->pVg; + tmq_t* tmq = pParam->tmq; if (code != 0) { printf("msg discard\n"); - tsem_post(&pParam->rspSem); + if (pParam->epoch == tmq->epoch) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } return 0; } - SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp)); + int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; + int32_t tmqEpoch = atomic_load_32(&tmq->epoch); + if (msgEpoch < tmqEpoch) { + printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch); + return 0; + } + + if (msgEpoch != tmqEpoch) { + printf("mismatch rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch); + } + + /*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/ + tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); if (pRsp == NULL) { - tsem_post(&pParam->rspSem); + printf("fail\n"); return -1; } - tDecodeSMqConsumeRsp(pMsg->pData, pRsp); + memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); + tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ - if (pRsp->numOfTopics == 0) { - /*printf("no data\n");*/ - free(pRsp); - tsem_post(&pParam->rspSem); + if (pRsp->consumeRsp.numOfTopics == 0) { + printf("no data\n"); + if (pParam->epoch == tmq->epoch) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } + taosFreeQitem(pRsp); return 0; } - *pParam->retMsg = (tmq_message_t*)pRsp; - pVg->currentOffset = pRsp->rspOffset; + pRsp->extra = pParam->pVg; + taosWriteQitem(tmq->mqueue, pRsp); + printf("poll in queue\n"); + /*pParam->rspMsg = (tmq_message_t*)pRsp;*/ + /*pVg->currentOffset = pRsp->consumeRsp.rspOffset;*/ + /*printf("rsp offset: %ld\n", rsp.rspOffset);*/ /*printf("-----msg begin----\n");*/ - tsem_post(&pParam->rspSem); /*printf("\n-----msg end------\n");*/ return 0; } +bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { + bool set = false; + int32_t sz = taosArrayGetSize(pRsp->topics); + if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); + tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic topic = {0}; + SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); + topic.topicName = strdup(pTopicEp->topic); + int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); + topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); + for (int32_t j = 0; j < vgSz; j++) { + SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + SMqClientVg clientVg = { + .pollCnt = 0, + .currentOffset = pVgEp->offset, + .vgId = pVgEp->vgId, + .epSet = pVgEp->epSet, + .vgStatus = TMQ_VG_STATUS__IDLE, + }; + taosArrayPush(topic.vgs, &clientVg); + set = true; + } + taosArrayPush(tmq->clientTopics, &topic); + } + atomic_store_32(&tmq->epoch, epoch); + return set; +} + int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; if (code != 0) { - printf("get topic endpoint error, not ready, wait:%d\n", pParam->wait); - if (pParam->wait) { - tsem_post(&tmq->rspSem); + printf("get topic endpoint error, not ready, wait:%d\n", pParam->sync); + if (pParam->sync) { + tsem_post(&pParam->rspSem); } return 0; } tscDebug("tmq ask ep cb called"); - bool set = false; - SMqCMGetSubEpRsp rsp; - tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); - int32_t sz = taosArrayGetSize(rsp.topics); - // TODO: lock - /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ - /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ - if (rsp.epoch != tmq->epoch) { - // TODO - if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); - tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); - for (int32_t i = 0; i < sz; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i); - topic.topicName = strdup(pTopicEp->topic); - int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); - topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); - for (int32_t j = 0; j < vgSz; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - // clang-format off - SMqClientVg clientVg = { - .pollCnt = 0, - .currentOffset = pVgEp->offset, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet - }; - // clang-format on - taosArrayPush(topic.vgs, &clientVg); - set = true; - } - taosArrayPush(tmq->clientTopics, &topic); + if (pParam->sync) { + SMqRspHead* head = pMsg->pData; + SMqCMGetSubEpRsp rsp; + tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); + /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ + /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ + int32_t epoch = atomic_load_32(&tmq->epoch); + if (head->epoch > epoch && tmqUpdateEp(tmq, head->epoch, &rsp)) { + atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY); } - tmq->epoch = rsp.epoch; - } - if (set) { - atomic_store_64(&tmq->status, 1); - } - // unlock - /*tsem_post(&tmq->rspSem);*/ - if (pParam->wait) { - tsem_post(&tmq->rspSem); + tsem_post(&pParam->rspSem); + tDeleteSMqCMGetSubEpRsp(&rsp); + } else { + tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); + tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp); + taosWriteQitem(tmq->mqueue, pRsp); } - tDeleteSMqCMGetSubEpRsp(&rsp); return 0; } -int32_t tmqAskEp(tmq_t* tmq, bool wait) { +int32_t tmqAskEp(tmq_t* tmq, bool sync) { + printf("ask ep sync %d\n", sync); int32_t tlen = sizeof(SMqCMGetSubEpReq); SMqCMGetSubEpReq* buf = malloc(tlen); if (buf == NULL) { @@ -722,7 +760,11 @@ int32_t tmqAskEp(tmq_t* tmq, bool wait) { goto END; } - pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; + pRequest->body.requestMsg = (SDataBuf){ + .pData = buf, + .len = tlen, + .handle = NULL, + }; SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam)); if (pParam == NULL) { @@ -730,7 +772,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool wait) { goto END; } pParam->tmq = tmq; - pParam->wait = wait; + pParam->sync = sync; + tsem_init(&pParam->rspSem, 0, 0); SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->requestObjRefId = 0; @@ -743,7 +786,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool wait) { asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); END: - if (wait) tsem_wait(&tmq->rspSem); + if (sync) tsem_wait(&pParam->rspSem); return 0; } @@ -791,6 +834,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie pReq->blockingTime = blocking_time; pReq->consumerId = tmq->consumerId; + pReq->epoch = tmq->epoch; pReq->currentOffset = reqOffset; pReq->head.vgId = htonl(pVg->vgId); @@ -798,11 +842,158 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie return pReq; } +void tmqClearUnhandleMsg(tmq_t* tmq) { + tmq_message_t* msg; + while (1) { + taosGetQitem(tmq->qall, (void**)&msg); + if (msg) + taosFreeQitem(msg); + else + break; + } + + taosReadAllQitems(tmq->mqueue, tmq->qall); + while (1) { + taosGetQitem(tmq->qall, (void**)&msg); + if (msg) + taosFreeQitem(msg); + else + break; + } +} + +int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { + printf("call poll\n"); + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); + if (vgStatus != TMQ_VG_STATUS__IDLE) { + continue; + } + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); + if (pReq == NULL) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + // TODO: out of mem + return -1; + } + SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); + if (param == NULL) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + // TODO: out of mem + return -1; + } + param->tmq = tmq; + param->pVg = pVg; + param->epoch = tmq->epoch; + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){ + .pData = pReq, + .len = sizeof(SMqConsumeReq), + .handle = NULL, + }; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = param; + sendInfo->fp = tmqPollCb; + + int64_t transporterId = 0; + printf("send poll\n"); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + pVg->pollCnt++; + tmq->pollCnt++; + } + } + return 0; +} + +// return +int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) { + if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) { + printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch); + if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) { + tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp); + tmqClearUnhandleMsg(tmq); + *pReset = true; + } else { + *pReset = false; + } + } else { + return -1; + } + return 0; +} + +tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { + while (1) { + tmq_message_t* rspMsg = NULL; + taosGetQitem(tmq->qall, (void**)&rspMsg); + if (rspMsg == NULL) { + break; + } + + if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { + printf("handle poll rsp %d\n", rspMsg->head.mqMsgType); + if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) { + printf("epoch match\n"); + SMqClientVg* pVg = rspMsg->extra; + pVg->currentOffset = rspMsg->consumeRsp.rspOffset; + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + return rspMsg; + } else { + printf("epoch mismatch\n"); + taosFreeQitem(rspMsg); + } + } else { + printf("handle ep rsp %d\n", rspMsg->head.mqMsgType); + bool reset = false; + tmqHandleRes(tmq, rspMsg, &reset); + taosFreeQitem(rspMsg); + if (pollIfReset && reset) { + printf("reset and repoll\n"); + tmqPollImpl(tmq, blockingTime); + } + } + } + return NULL; +} + tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { - tmq_message_t* tmq_message = NULL; + tmq_message_t* rspMsg = NULL; + int64_t startTime = taosGetTimestampMs(); + // TODO: put into another thread or delayed queue int64_t status = atomic_load_64(&tmq->status); - tmqAskEp(tmq, status == 0); + tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); + + taosGetQitem(tmq->qall, (void**)&rspMsg); + if (rspMsg == NULL) { + taosReadAllQitems(tmq->mqueue, tmq->qall); + } + tmqHandleAllRsp(tmq, blocking_time, false); + + tmqPollImpl(tmq, blocking_time); + + while (1) { + /*printf("cycle\n");*/ + taosReadAllQitems(tmq->mqueue, tmq->qall); + rspMsg = tmqHandleAllRsp(tmq, blocking_time, true); + if (rspMsg) { + return rspMsg; + } + if (blocking_time != 0) { + int64_t endTime = taosGetTimestampMs(); + if (endTime - startTime > blocking_time) { + printf("normal exit\n"); + return NULL; + } + } + } +} + +#if 0 if (blocking_time <= 0) blocking_time = 1; if (blocking_time > 1000) blocking_time = 1000; @@ -834,7 +1025,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { return NULL; } - SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam)); + SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); if (param == NULL) { ASSERT(false); usleep(blocking_time * 1000); @@ -846,7 +1037,11 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tsem_init(¶m->rspSem, 0, 0); SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq), .handle = NULL}; + pRequest->body.requestMsg = (SDataBuf){ + .pData = pReq, + .len = sizeof(SMqConsumeReq), + .handle = NULL, + }; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->requestObjRefId = 0; @@ -886,6 +1081,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { /*return pRequest;*/ } +#endif #if 0 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { @@ -929,9 +1125,9 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; - SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; tDeleteSMqConsumeRsp(pRsp); - free(tmq_message); + taosFreeQitem(tmq_message); } tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index db84f87b66c77fd636eb6df1d04efe6c82d108f8..d66895888b60d4c7d265079513ab5415b3ed9507 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -177,24 +177,24 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { - uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr()); return -1; } - if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) { - if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) { - uError("failed to load from config file:%s since %s\n", cfgFile, terrstr()); - return -1; + if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) { + if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) { + uError("failed to load from config file:%s since %s", cfgFile, terrstr()); + return 0; } } if (cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile) != 0) { - uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + uError("failed to load from env file:%s since %s", envFile, terrstr()); return -1; } if (cfgLoad(pCfg, CFG_STYPE_ENV_VAR, NULL) != 0) { - uError("failed to load from global env variables since %s\n", terrstr()); + uError("failed to load from global env variables since %s", terrstr()); return -1; } @@ -271,7 +271,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddTimezone(pCfg, "timezone", tsTimezone) != 0) return -1; if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; - if (cfgAddBool(pCfg, "enableCoreFile", 0, 1) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1; if (cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; @@ -438,8 +438,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi if (pCfg == NULL) return -1; if (tsc) { + tscEmbeddedInUtil = 0; if (taosAddClientLogCfg(pCfg) != 0) return -1; } else { + tscEmbeddedInUtil = 1; if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddServerLogCfg(pCfg) != 0) return -1; } @@ -450,7 +452,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi return -1; } - if (cfgLoadArray(pCfg, pArgs) != 0) { + if (cfgLoadFromArray(pCfg, pArgs) != 0) { uError("failed to load cfg from array since %s", terrstr()); cfgCleanup(pCfg); return -1; @@ -465,8 +467,14 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); + if (taosMkDir(tsLogDir) != 0) { + uError("failed to create dir:%s since %s", tsLogDir, terrstr()); + cfgCleanup(pCfg); + return -1; + } + if (taosInitLog(logname, logFileNum) != 0) { - printf("failed to init log file since %s\n", terrstr()); + uError("failed to init log file since %s", terrstr()); cfgCleanup(pCfg); return -1; } @@ -497,7 +505,7 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU return -1; } - if (cfgLoadArray(tsCfg, pArgs) != 0) { + if (cfgLoadFromArray(tsCfg, pArgs) != 0) { uError("failed to load cfg from array since %s", terrstr()); cfgCleanup(tsCfg); return -1; @@ -512,6 +520,16 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU } taosSetSystemCfg(tsCfg); + if (taosMkDir(tsTempDir) != 0) { + uError("failed to create dir:%s since %s", tsTempDir, terrstr()); + return -1; + } + + if (!tsc && taosMkDir(tsDataDir) != 0) { + uError("failed to create dir:%s since %s", tsDataDir, terrstr()); + return -1; + } + cfgDumpCfg(tsCfg, tsc, false); return 0; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8a3cd0a7183b935da6d81f6f6651c954942f4a3f..d8c850c6afa9588c21f3151f07513e0268a918a8 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -270,7 +270,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR int32_t rspNum = 0; if (tDecodeI32(&decoder, &rspNum) < 0) return -1; if (pBatchRsp->rsps == NULL) { - pBatchRsp->rsps = taosArrayInit(rspNum, sizeof(SClientHbReq)); + pBatchRsp->rsps = taosArrayInit(rspNum, sizeof(SClientHbRsp)); } for (int32_t i = 0; i < rspNum; i++) { SClientHbRsp rsp = {0}; @@ -1421,6 +1421,7 @@ int32_t tSerializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbId) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; tEndEncode(&encoder); @@ -1435,6 +1436,7 @@ int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; tEndDecode(&decoder); @@ -1527,7 +1529,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (pRsp->vgNum <= 0) { return 0; } - + pRsp->pVgroupInfos = taosArrayInit(pRsp->vgNum, sizeof(SVgroupInfo)); if (pRsp->pVgroupInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index f32698c98ad7179f2661c85fec2df3906aa76dc0..0c1bafbeb86f9dc04521d7245193c620b6f42dda 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -49,16 +49,16 @@ const int32_t TYPE_BYTES[15] = { } \ } while (0) -static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int8_t *data = (int8_t *)pData; *min = INT64_MAX; *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (data[i] == TSDB_DATA_BOOL_NULL) { (*numOfNull) += 1; @@ -76,9 +76,9 @@ static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) { (*numOfNull) += 1; @@ -122,9 +122,9 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) { (*numOfNull) += 1; @@ -133,15 +133,14 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); } - } static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint16_t *data = (uint16_t *)pData; - uint64_t _min = UINT64_MAX; - uint64_t _max = 0; - uint64_t _sum = 0; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; *minIndex = 0; *maxIndex = 0; @@ -169,9 +168,9 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) { (*numOfNull) += 1; @@ -182,12 +181,12 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in } } -static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint32_t *data = (uint32_t *)pData; - uint64_t _min = UINT64_MAX; - uint64_t _max = 0; - uint64_t _sum = 0; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; *minIndex = 0; *maxIndex = 0; @@ -208,16 +207,16 @@ static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, in *sum = _sum; } -static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *data = (int64_t *)pData; *min = INT64_MAX; *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) { (*numOfNull) += 1; @@ -228,12 +227,12 @@ static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, in } } -static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint64_t *data = (uint64_t *)pData; - uint64_t _min = UINT64_MAX; - uint64_t _max = 0; - uint64_t _sum = 0; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; *minIndex = 0; *maxIndex = 0; @@ -254,91 +253,91 @@ static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, in *sum = _sum; } -static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { - float *data = (float *)pData; - float fmin = FLT_MAX; - float fmax = -FLT_MAX; - double dsum = 0; - *minIndex = 0; - *maxIndex = 0; - +static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + float *data = (float *)pData; + float fmin = FLT_MAX; + float fmax = -FLT_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { - if ((*(uint32_t*)&(data[i])) == TSDB_DATA_FLOAT_NULL) { + if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) { (*numOfNull) += 1; continue; } - float fv = GET_FLOAT_VAL((const char*)&(data[i])); + float fv = GET_FLOAT_VAL((const char *)&(data[i])); dsum += fv; if (fmin > fv) { fmin = fv; *minIndex = i; } - + if (fmax < fv) { fmax = fv; *maxIndex = i; } } - + SET_DOUBLE_VAL(sum, dsum); SET_DOUBLE_VAL(max, fmax); SET_DOUBLE_VAL(min, fmin); } -static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { double *data = (double *)pData; - double dmin = DBL_MAX; - double dmax = -DBL_MAX; - double dsum = 0; - *minIndex = 0; - *maxIndex = 0; - + double dmin = DBL_MAX; + double dmax = -DBL_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { - if ((*(uint64_t*)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { + if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { (*numOfNull) += 1; continue; } - + double dv = 0; - dv = GET_DOUBLE_VAL((const char*)&(data[i])); + dv = GET_DOUBLE_VAL((const char *)&(data[i])); dsum += dv; if (dmin > dv) { dmin = dv; *minIndex = i; } - + if (dmax < dv) { dmax = dv; *maxIndex = i; } } - + SET_DOUBLE_PTR(sum, &dsum); SET_DOUBLE_PTR(max, &dmax); SET_DOUBLE_PTR(min, &dmin); } -static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { - const char* data = pData; +static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + const char *data = pData; assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull(data, TSDB_DATA_TYPE_BINARY)) { (*numOfNull) += 1; } - + data += varDataTLen(data); } - + *sum = 0; *max = 0; *min = 0; @@ -346,19 +345,19 @@ static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, in *maxIndex = 0; } -static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { - const char* data = pData; +static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + const char *data = pData; assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull(data, TSDB_DATA_TYPE_NCHAR)) { (*numOfNull) += 1; } - + data += varDataTLen(data); } - + *sum = 0; *max = 0; *min = 0; @@ -367,21 +366,28 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i } tDataTypeDescriptor tDataTypes[15] = { - {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, - {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, - {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, - {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_i16}, - {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt, tsDecompressInt, getStatics_i32}, - {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_i64}, - {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f}, - {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d}, - {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, - {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64}, - {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, - {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_u8}, - {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_u16}, - {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, - {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, + {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, + {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, + {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, + getStatics_i8}, + {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint, + tsDecompressSmallint, getStatics_i16}, + {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt, tsDecompressInt, getStatics_i32}, + {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint, tsDecompressBigint, + getStatics_i64}, + {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f}, + {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d}, + {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, + {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, + tsDecompressTimestamp, getStatics_i64}, + {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, + getStatics_u8}, + {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, + tsDecompressSmallint, getStatics_u16}, + {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, + {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, + getStatics_u64}, }; char tTokenTypeSwitcher[13] = { @@ -400,10 +406,10 @@ char tTokenTypeSwitcher[13] = { TSDB_DATA_TYPE_NCHAR, // TK_NCHAR }; -float floatMin = -FLT_MAX, floatMax = FLT_MAX; +float floatMin = -FLT_MAX, floatMax = FLT_MAX; double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; -FORCE_INLINE void* getDataMin(int32_t type) { +FORCE_INLINE void *getDataMin(int32_t type) { switch (type) { case TSDB_DATA_TYPE_FLOAT: return &floatMin; @@ -414,7 +420,7 @@ FORCE_INLINE void* getDataMin(int32_t type) { } } -FORCE_INLINE void* getDataMax(int32_t type) { +FORCE_INLINE void *getDataMax(int32_t type) { switch (type) { case TSDB_DATA_TYPE_FLOAT: return &floatMax; @@ -425,17 +431,15 @@ FORCE_INLINE void* getDataMax(int32_t type) { } } -bool isValidDataType(int32_t type) { - return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; -} +bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; } -void setVardataNull(void* val, int32_t type) { +void setVardataNull(void *val, int32_t type) { if (type == TSDB_DATA_TYPE_BINARY) { varDataSetLen(val, sizeof(int8_t)); - *(uint8_t*) varDataVal(val) = TSDB_DATA_BINARY_NULL; + *(uint8_t *)varDataVal(val) = TSDB_DATA_BINARY_NULL; } else if (type == TSDB_DATA_TYPE_NCHAR) { varDataSetLen(val, sizeof(int32_t)); - *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL; + *(uint32_t *)varDataVal(val) = TSDB_DATA_NCHAR_NULL; } else { assert(0); } @@ -533,9 +537,8 @@ static SBinaryNullT nullBinary = {1, TSDB_DATA_BINARY_NULL}; static SNCharNullT nullNchar = {4, TSDB_DATA_NCHAR_NULL}; static const void *nullValues[] = { - &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, - &nullFloat, &nullDouble, &nullBinary, &nullBigInt, &nullNchar, - &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu, + &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, &nullFloat, &nullDouble, + &nullBinary, &nullBigInt, &nullNchar, &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu, }; const void *getNullValue(int32_t type) { @@ -590,7 +593,7 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2); break; case TSDB_DATA_TYPE_UTINYINT: - *((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2); + *((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2); break; case TSDB_DATA_TYPE_SMALLINT: *((int16_t *)dst) = GET_INT16_VAL(s1) + GET_INT16_VAL(s2); @@ -629,14 +632,14 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { } } -void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void *buf) { switch (type) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: { TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); break; } - + case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { @@ -652,19 +655,19 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); break; } - + case TSDB_DATA_TYPE_FLOAT: { TSWAP(*(float *)(pLeft), *(float *)(pRight), float); break; } - + case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: { TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); break; } - + default: { memcpy(buf, pLeft, size); memcpy(pLeft, pRight, size); diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index ceef12e986105ad42b1322db26c20a19a760f740..e885d842d77442edc72a83b86458770001b52a06 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -35,9 +35,9 @@ assert(0); \ } while (0) -int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* isSigned) { +int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value, bool *isSigned) { errno = 0; - char* endPtr = NULL; + char *endPtr = NULL; int32_t index = 0; @@ -63,15 +63,15 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* *isSigned = specifiedSign || (val <= INT64_MAX); if (*isSigned) { - *value = (z[0] == '-')? -val:val; + *value = (z[0] == '-') ? -val : val; } else { - *(uint64_t*) value = val; + *(uint64_t *)value = val; } return 0; } -void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) { +void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) { int32_t ret = 0; memset(pVar, 0, sizeof(SVariant)); @@ -90,7 +90,7 @@ void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_INT:{ + case TSDB_DATA_TYPE_INT: { bool sign = true; int32_t base = 10; @@ -104,11 +104,11 @@ void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) { ret = toInteger(z, n, base, &pVar->i, &sign); if (ret != 0) { - pVar->nType = -1; // -1 means error type + pVar->nType = -1; // -1 means error type return; } - pVar->nType = (sign)? TSDB_DATA_TYPE_BIGINT:TSDB_DATA_TYPE_UBIGINT; + pVar->nType = (sign) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_UBIGINT; break; } case TSDB_DATA_TYPE_DOUBLE: @@ -123,15 +123,15 @@ void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) { } case TSDB_DATA_TYPE_TIMESTAMP: { assert(0); - pVar->i = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); - break; - } - + pVar->i = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); + break; + } + default: { // nType == 0 means the null value type = TSDB_DATA_TYPE_NULL; } } - + pVar->nType = type; } @@ -196,13 +196,13 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin pVar->d = GET_FLOAT_VAL(pz); break; } - case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length + case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; - + break; } case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method @@ -211,18 +211,18 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin pVar->nLen = (int32_t)len; break; } - + default: pVar->i = GET_INT32_VAL(pz); pVar->nLen = tDataTypes[TSDB_DATA_TYPE_INT].bytes; } - + pVar->nType = type; } void taosVariantDestroy(SVariant *pVar) { if (pVar == NULL) return; - + if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) { tfree(pVar->pz); pVar->nLen = 0; @@ -231,8 +231,8 @@ void taosVariantDestroy(SVariant *pVar) { // NOTE: this is only for string array if (pVar->nType == TSDB_DATA_TYPE_POINTER_ARRAY) { size_t num = taosArrayGetSize(pVar->arr); - for(size_t i = 0; i < num; i++) { - void* p = taosArrayGetP(pVar->arr, i); + for (size_t i = 0; i < num; i++) { + void *p = taosArrayGetP(pVar->arr, i); free(p); } taosArrayDestroy(pVar->arr); @@ -250,11 +250,11 @@ bool taosVariantIsValid(SVariant *pVar) { void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; - + pDst->nType = pSrc->nType; if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; - char* p = realloc(pDst->pz, len); + char *p = realloc(pDst->pz, len); assert(p); memset(p, 0, len); @@ -263,28 +263,27 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { memcpy(pDst->pz, pSrc->pz, pSrc->nLen); pDst->nLen = pSrc->nLen; return; - } if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) { pDst->i = pSrc->i; } else if (pSrc->nType == TSDB_DATA_TYPE_POINTER_ARRAY) { // this is only for string array size_t num = taosArrayGetSize(pSrc->arr); - pDst->arr = taosArrayInit(num, sizeof(char*)); - for(size_t i = 0; i < num; i++) { - char* p = (char*)taosArrayGetP(pSrc->arr, i); - char* n = strdup(p); + pDst->arr = taosArrayInit(num, sizeof(char *)); + for (size_t i = 0; i < num; i++) { + char *p = (char *)taosArrayGetP(pSrc->arr, i); + char *n = strdup(p); taosArrayPush(pDst->arr, &n); } } else if (pSrc->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { - size_t num = taosArrayGetSize(pSrc->arr); - pDst->arr = taosArrayInit(num, sizeof(int64_t)); - pDst->nLen = pSrc->nLen; - assert(pSrc->nLen == num); - for(size_t i = 0; i < num; i++) { - int64_t *p = taosArrayGet(pSrc->arr, i); - taosArrayPush(pDst->arr, p); - } + size_t num = taosArrayGetSize(pSrc->arr); + pDst->arr = taosArrayInit(num, sizeof(int64_t)); + pDst->nLen = pSrc->nLen; + assert(pSrc->nLen == num); + for (size_t i = 0; i < num; i++) { + int64_t *p = taosArrayGet(pSrc->arr, i); + taosArrayPush(pDst->arr, p); + } } if (pDst->nType != TSDB_DATA_TYPE_POINTER_ARRAY && pDst->nType != TSDB_DATA_TYPE_VALUE_ARRAY) { @@ -292,7 +291,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { } } -int32_t taosVariantCompare(const SVariant* p1, const SVariant* p2) { +int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) { if (p1->nType == TSDB_DATA_TYPE_NULL && p2->nType == TSDB_DATA_TYPE_NULL) { return 0; } @@ -309,39 +308,39 @@ int32_t taosVariantCompare(const SVariant* p1, const SVariant* p2) { if (p1->nLen == p2->nLen) { return memcmp(p1->pz, p2->pz, p1->nLen); } else { - return p1->nLen > p2->nLen? 1:-1; + return p1->nLen > p2->nLen ? 1 : -1; } } else if (p1->nType == TSDB_DATA_TYPE_FLOAT || p1->nType == TSDB_DATA_TYPE_DOUBLE) { if (p1->d == p2->d) { return 0; } else { - return p1->d > p2->d? 1:-1; + return p1->d > p2->d ? 1 : -1; } } else if (IS_UNSIGNED_NUMERIC_TYPE(p1->nType)) { if (p1->u == p2->u) { return 0; } else { - return p1->u > p2->u? 1:-1; + return p1->u > p2->u ? 1 : -1; } } else { if (p1->i == p2->i) { return 0; } else { - return p1->i > p2->i? 1:-1; + return p1->i > p2->i ? 1 : -1; } } } int32_t taosVariantToString(SVariant *pVar, char *dst) { if (pVar == NULL || dst == NULL) return 0; - + switch (pVar->nType) { case TSDB_DATA_TYPE_BINARY: { int32_t len = sprintf(dst, "\'%s\'", pVar->pz); assert(len <= pVar->nLen + sizeof("\'") * 2); // two more chars return len; } - + case TSDB_DATA_TYPE_NCHAR: { dst[0] = '\''; taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); @@ -350,7 +349,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) { dst[len + 1] = 0; return len + 1; } - + case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -359,7 +358,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) { case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: return sprintf(dst, "%d", (int32_t)pVar->i); - + case TSDB_DATA_TYPE_BIGINT: return sprintf(dst, "%" PRId64, pVar->i); case TSDB_DATA_TYPE_UBIGINT: @@ -367,7 +366,7 @@ int32_t taosVariantToString(SVariant *pVar, char *dst) { case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: return sprintf(dst, "%.9lf", pVar->d); - + default: return 0; } @@ -399,27 +398,27 @@ static FORCE_INLINE int32_t wcsconvertToBoolImpl(wchar_t *pstr, int32_t len) { static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { const int32_t INITIAL_ALLOC_SIZE = 40; - char * pBuf = NULL; + char *pBuf = NULL; // it is a in-place convert type for SVariant, local buffer is needed if (*pDest == pVariant->pz) { pBuf = calloc(1, INITIAL_ALLOC_SIZE); } - + if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { size_t newSize = pVariant->nLen * TSDB_NCHAR_SIZE; if (pBuf != NULL) { if (newSize >= INITIAL_ALLOC_SIZE) { pBuf = realloc(pBuf, newSize + 1); } - + taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf); free(pVariant->wpz); pBuf[newSize] = 0; } else { taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest); } - + } else { if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) { sprintf(pBuf == NULL ? *pDest : pBuf, "%" PRId64, pVariant->i); @@ -431,26 +430,26 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { setNull(pBuf == NULL ? *pDest : pBuf, TSDB_DATA_TYPE_BINARY, 0); } } - + if (pBuf != NULL) { *pDest = pBuf; } - + *pDestSize = (int32_t)strlen(*pDest); return 0; } static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { char tmpBuf[40] = {0}; - - char * pDst = tmpBuf; + + char *pDst = tmpBuf; int32_t nLen = 0; // convert the number to string, than convert it to wchar string. if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) { nLen = sprintf(pDst, "%" PRId64, pVariant->i); } else if (IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { - nLen = sprintf(pDst, "%"PRIu64, pVariant->u); + nLen = sprintf(pDst, "%" PRIu64, pVariant->u); } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) { nLen = sprintf(pDst, "%lf", pVariant->d); } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { @@ -459,10 +458,10 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL) { nLen = sprintf(pDst, "%s", (pVariant->i == TSDB_TRUE) ? "TRUE" : "FALSE"); } - + if (*pDest == pVariant->pz) { wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); - bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); + bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); if (!ret) { tfree(pWStr); return -1; @@ -472,14 +471,14 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { free(pVariant->wpz); } - + pVariant->wpz = pWStr; *pDestSize = twcslen(pVariant->wpz); - + // shrink the allocate memory, no need to check here. - char* tmp = realloc(pVariant->wpz, (*pDestSize + 1)*TSDB_NCHAR_SIZE); + char *tmp = realloc(pVariant->wpz, (*pDestSize + 1) * TSDB_NCHAR_SIZE); assert(tmp != NULL); - + pVariant->wpz = (wchar_t *)tmp; } else { int32_t output = 0; @@ -493,21 +492,22 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { *pDestSize = output; } } - + return 0; } static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *value) { -// SToken stoken = {.z = pStr, .n = len}; -// if (TK_ILLEGAL == tGetNumericStringType(&stoken)) { -// return -1; -// } -// -// *value = strtod(pStr, NULL); + // SToken stoken = {.z = pStr, .n = len}; + // if (TK_ILLEGAL == tGetNumericStringType(&stoken)) { + // return -1; + // } + // + // *value = strtod(pStr, NULL); return 0; } -static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr, bool *converted) { +static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result, int32_t type, bool issigned, + bool releaseVariantPtr, bool *converted) { if (pVariant->nType == TSDB_DATA_TYPE_NULL) { setNull((char *)result, type, tDataTypes[type].bytes); return 0; @@ -645,7 +645,7 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) { if ((ret = convertToBoolImpl(pVariant->pz, pVariant->nLen)) < 0) { return ret; } - + *pDest = ret; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { int32_t ret = 0; @@ -656,7 +656,7 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) { } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) { *pDest = TSDB_DATA_BOOL_NULL; } - + assert(*pDest == TSDB_TRUE || *pDest == TSDB_FALSE || *pDest == TSDB_DATA_BOOL_NULL); return 0; } @@ -665,11 +665,12 @@ static int32_t convertToBool(SVariant *pVariant, int64_t *pDest) { * transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType * to column type defined in schema */ -int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo) { +int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, + char *extInfo) { if (converted) { *converted = false; } - + if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) { return -1; } @@ -686,25 +687,25 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc *(int8_t *)payload = (int8_t)result; break; } - + case TSDB_DATA_TYPE_TINYINT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { SET_EXT_INFO(converted, result, INT8_MIN + 1, INT8_MAX, extInfo); return -1; } - *((int8_t *)payload) = (int8_t) result; + *((int8_t *)payload) = (int8_t)result; break; } case TSDB_DATA_TYPE_UTINYINT: { - if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { + if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { SET_EXT_INFO(converted, result, 0, UINT8_MAX - 1, extInfo); return -1; } - *((uint8_t *)payload) = (uint8_t) result; + *((uint8_t *)payload) = (uint8_t)result; break; } - + case TSDB_DATA_TYPE_SMALLINT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { SET_EXT_INFO(converted, result, INT16_MIN + 1, INT16_MAX, extInfo); @@ -722,7 +723,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc *((uint16_t *)payload) = (uint16_t)result; break; } - + case TSDB_DATA_TYPE_INT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { SET_EXT_INFO(converted, result, INT32_MIN + 1, INT32_MAX, extInfo); @@ -740,7 +741,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc *((uint32_t *)payload) = (uint32_t)result; break; } - + case TSDB_DATA_TYPE_BIGINT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { SET_EXT_INFO(converted, (int64_t)result, INT64_MIN + 1, INT64_MAX, extInfo); @@ -775,19 +776,20 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc if (converted) { *converted = true; } - + if (value > FLT_MAX || value < -FLT_MAX) { SET_EXT_INFO(converted, value, -FLT_MAX, FLT_MAX, extInfo); return -1; } SET_FLOAT_VAL(payload, value); } - } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { + } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || + IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { if (converted) { *converted = true; } - - if (pVariant->i > FLT_MAX || pVariant->i < -FLT_MAX) { + + if (pVariant->i > FLT_MAX || pVariant->i < -FLT_MAX) { SET_EXT_INFO(converted, pVariant->i, -FLT_MAX, FLT_MAX, extInfo); return -1; } @@ -797,12 +799,12 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc if (converted) { *converted = true; } - - if (pVariant->d > FLT_MAX || pVariant->d < -FLT_MAX) { + + if (pVariant->d > FLT_MAX || pVariant->d < -FLT_MAX) { SET_EXT_INFO(converted, pVariant->d, -FLT_MAX, FLT_MAX, extInfo); return -1; } - + SET_FLOAT_VAL(payload, pVariant->d); } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) { *((uint32_t *)payload) = TSDB_DATA_FLOAT_NULL; @@ -831,7 +833,8 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc SET_DOUBLE_VAL(payload, value); } - } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { + } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || + IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { SET_DOUBLE_VAL(payload, pVariant->i); } else if (IS_FLOAT_TYPE(pVariant->nType)) { SET_DOUBLE_VAL(payload, pVariant->d); @@ -847,11 +850,11 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc break; } - + case TSDB_DATA_TYPE_BINARY: { if (!includeLengthPrefix) { if (pVariant->nType == TSDB_DATA_TYPE_NULL) { - *(uint8_t*) payload = TSDB_DATA_BINARY_NULL; + *(uint8_t *)payload = TSDB_DATA_BINARY_NULL; } else { if (pVariant->nType != TSDB_DATA_TYPE_BINARY) { toBinary(pVariant, &payload, &pVariant->nLen); @@ -918,11 +921,11 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc assert(p == varDataVal(payload)); } } - + break; } } - + return 0; } @@ -944,13 +947,13 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { if (pVariant == NULL || pVariant->nType == 0) { // value is not set return 0; } - + switch (type) { case TSDB_DATA_TYPE_BOOL: { // bool if (convertToBool(pVariant, &pVariant->i) < 0) { return -1; } - + pVariant->nType = type; break; } @@ -971,7 +974,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { free(pVariant->pz); return -1; } - + free(pVariant->pz); pVariant->d = v; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { @@ -981,14 +984,14 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { free(pVariant->pz); return -1; } - + free(pVariant->pz); pVariant->d = v; } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { - double tmp = (double) pVariant->i; + double tmp = (double)pVariant->i; pVariant->d = tmp; } - + pVariant->nType = TSDB_DATA_TYPE_DOUBLE; break; } @@ -1009,6 +1012,6 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { break; } } - + return 0; } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index dfa2b747556989bf7cde63bc038e7c1d97444e84..157bad26a6a33493920e1cbb3875f91ffbca74ec 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -114,7 +114,6 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg; - /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/ pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9165fa2264725253f9a1082f47ae26ae27a0f7d4..974f5fc98237a8f40995e88377053771947f1cba 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -940,43 +940,51 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { char *p = strchr(usedbReq.db, '.'); if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); + code = 0; } else { pDb = mndAcquireDb(pMnode, usedbReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto USE_DB_OVER; - } - pUser = mndAcquireUser(pMnode, pReq->user); - if (pUser == NULL) { - goto USE_DB_OVER; - } + memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); + usedbRsp.uid = usedbReq.dbId; + usedbRsp.vgVersion = usedbReq.vgVersion; - if (mndCheckUseDbAuth(pUser, pDb) != 0) { - goto USE_DB_OVER; - } + mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); + } else { + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto USE_DB_OVER; + } - usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); - if (usedbRsp.pVgroupInfos == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto USE_DB_OVER; - } + if (mndCheckUseDbAuth(pUser, pDb) != 0) { + goto USE_DB_OVER; + } - if (usedbReq.vgVersion < pDb->vgVersion) { - mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); - } + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } + + if (usedbReq.vgVersion < pDb->vgVersion || usedbReq.dbId != pDb->uid) { + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + } - memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); - usedbRsp.uid = pDb->uid; - usedbRsp.vgVersion = pDb->vgVersion; - usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); - usedbRsp.hashMethod = pDb->hashMethod; + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + code = 0; + } } int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + code = -1; goto USE_DB_OVER; } @@ -984,7 +992,6 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { pReq->pCont = pRsp; pReq->contLen = contLen; - code = 0; USE_DB_OVER: if (code != 0) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 5ae5cd39fb659d5c0ef1baa5a44330f0e8a4b62e..ec48b25d2ad2c1033915cd1ebf875e7058e9d6af 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -270,9 +270,8 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(rsp.cgroup, pReq->cgroup); rsp.consumerId = consumerId; - rsp.epoch = pConsumer->epoch; - if (epoch != rsp.epoch) { - mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, rsp.epoch); + if (epoch != pConsumer->epoch) { + mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, pConsumer->epoch); SArray *pTopics = pConsumer->currentTopics; int32_t sz = taosArrayGetSize(pTopics); rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); @@ -308,13 +307,16 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { mndReleaseSubscribe(pMnode, pSub); } } - int32_t tlen = tEncodeSMqCMGetSubEpRsp(NULL, &rsp); + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp); void *buf = rpcMallocCont(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - void *abuf = buf; + ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; + ((SMqRspHead *)buf)->epoch = pConsumer->epoch; + + void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); tDeleteSMqCMGetSubEpRsp(&rsp); mndReleaseConsumer(pMnode, pConsumer); diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index d6a4c76c6220b8b0cc796a48497ea45141c5a4d1..1cd7593846716f54b7cbb491dc7e207280821c80 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -18,227 +18,68 @@ #include "mndCluster.h" #include "mndSync.h" #include "tbuffer.h" -#include "tversion.h" +#include "tjson.h" +#include "thttp.h" #define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 -static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); } - -static void mndCloseObject(SBufferWriter* bw) { - size_t len = tbufTell(bw); - if (tbufGetData(bw, false)[len - 1] == ',') { - tbufWriteCharAt(bw, len - 1, '}'); - } else { - tbufWriteChar(bw, '}'); - } -} - -static void mndWriteString(SBufferWriter* bw, const char* str) { - tbufWriteChar(bw, '"'); - tbufWrite(bw, str, strlen(str)); - tbufWriteChar(bw, '"'); -} - -static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) { - mndWriteString(bw, k); - tbufWriteChar(bw, ':'); - char buf[32] = {0}; - sprintf(buf, "%" PRId64, v); - tbufWrite(bw, buf, strlen(buf)); - tbufWriteChar(bw, ','); -} - -static void mndAddStringField(SBufferWriter* bw, const char* k, const char* v) { - mndWriteString(bw, k); - tbufWriteChar(bw, ':'); - mndWriteString(bw, v); - tbufWriteChar(bw, ','); -} - -static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { - char* line = NULL; - size_t size = 0; - int32_t done = 0; - - // FILE* fp = fopen("/proc/cpuinfo", "r"); - TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - return; - } - - while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { - const char* v = strchr(line, ':') + 2; - mndAddStringField(bw, "cpuModel", v); - done |= 1; - } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { - const char* v = strchr(line, ':') + 2; - mndWriteString(bw, "numOfCpu"); - tbufWriteChar(bw, ':'); - tbufWrite(bw, v, strlen(v)); - tbufWriteChar(bw, ','); - done |= 2; - } - } - - if(line != NULL) free(line); - taosCloseFile(&pFile); -} - -static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { - char* line = NULL; - size_t size = 0; - - // FILE* fp = fopen("/etc/os-release", "r"); - TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - return; - } - - while ((size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "PRETTY_NAME", 11) == 0) { - const char* p = strchr(line, '=') + 1; - if (*p == '"') { - p++; - line[size - 2] = 0; - } - mndAddStringField(bw, "os", p); - break; - } - } - - if(line != NULL) free(line); - taosCloseFile(&pFile); -} - -static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { - char* line = NULL; - size_t size = 0; - - // FILE* fp = fopen("/proc/meminfo", "r"); - TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - return; - } - - while ((size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "MemTotal", 8) == 0) { - const char* p = strchr(line, ':') + 1; - while (*p == ' ') p++; - mndAddStringField(bw, "memory", p); - break; - } - } - - if(line != NULL) free(line); - taosCloseFile(&pFile); +static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) { + SMnodeLoad load = {0}; + if (mndGetLoad(pMnode, &load) != 0) return; + + tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode); + tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode); + tjsonAddDoubleToObject(pJson, "numOfVgroup", load.numOfVgroup); + tjsonAddDoubleToObject(pJson, "numOfDatabase", load.numOfDatabase); + tjsonAddDoubleToObject(pJson, "numOfSuperTable", load.numOfSuperTable); + tjsonAddDoubleToObject(pJson, "numOfChildTable", load.numOfChildTable); + tjsonAddDoubleToObject(pJson, "numOfColumn", load.numOfColumn); + tjsonAddDoubleToObject(pJson, "numOfPoint", load.totalPoints); + tjsonAddDoubleToObject(pJson, "totalStorage", load.totalStorage); + tjsonAddDoubleToObject(pJson, "compStorage", load.compStorage); } -static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) { +static char* mndBuildTelemetryReport(SMnode* pMnode) { + char tmp[4096] = {0}; STelemMgmt* pMgmt = &pMnode->telemMgmt; - mndAddStringField(bw, "version", version); - mndAddStringField(bw, "buildInfo", buildinfo); - mndAddStringField(bw, "gitInfo", gitinfo); - mndAddStringField(bw, "email", pMgmt->email); -} - -static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) { - SMnodeLoad load = {0}; - if (mndGetLoad(pMnode, &load) != 0) { - return; - } - mndAddIntField(bw, "numOfDnode", load.numOfDnode); - mndAddIntField(bw, "numOfMnode", load.numOfMnode); - mndAddIntField(bw, "numOfVgroup", load.numOfVgroup); - mndAddIntField(bw, "numOfDatabase", load.numOfDatabase); - mndAddIntField(bw, "numOfSuperTable", load.numOfSuperTable); - mndAddIntField(bw, "numOfChildTable", load.numOfChildTable); - mndAddIntField(bw, "numOfColumn", load.numOfColumn); - mndAddIntField(bw, "numOfPoint", load.totalPoints); - mndAddIntField(bw, "totalStorage", load.totalStorage); - mndAddIntField(bw, "compStorage", load.compStorage); -} - -static void mndSendTelemetryReport(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - SBufferWriter bw = tbufInitWriter(NULL, false); - int32_t code = -1; - char buf[128] = {0}; - SOCKET fd = 0; + SJson* pJson = tjsonCreateObject(); + if (pJson == NULL) return NULL; char clusterName[64] = {0}; - if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) { - goto SEND_OVER; - } - - mndBeginObject(&bw); - mndAddStringField(&bw, "instanceId", clusterName); - mndAddIntField(&bw, "reportVersion", 1); - mndAddOsInfo(pMnode, &bw); - mndAddCpuInfo(pMnode, &bw); - mndAddMemoryInfo(pMnode, &bw); - mndAddVersionInfo(pMnode, &bw); - mndAddRuntimeInfo(pMnode, &bw); - mndCloseObject(&bw); + mndGetClusterName(pMnode, clusterName, sizeof(clusterName)); + tjsonAddStringToObject(pJson, "instanceId", clusterName); + tjsonAddDoubleToObject(pJson, "reportVersion", 1); - uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); - if (ip == 0xffffffff) { - terrno = TAOS_SYSTEM_ERROR(errno); - mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr()); - goto SEND_OVER; + if (taosGetOsReleaseName(tmp, sizeof(tmp))) { + tjsonAddStringToObject(pJson, "os", tmp); } - fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); - if (fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); - goto SEND_OVER; - } - - const char* header = - "POST /report HTTP/1.1\n" - "Host: " TELEMETRY_SERVER - "\n" - "Content-Type: application/json\n" - "Content-Length: "; - if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) { - goto SEND_OVER; - } - - int32_t contLen = (int32_t)(tbufTell(&bw)); - sprintf(buf, "%d\n\n", contLen); - if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) { - goto SEND_OVER; - } - - const char* pCont = tbufGetData(&bw, false); - if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { - goto SEND_OVER; + int32_t numOfCores = 0; + if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores)) { + tjsonAddStringToObject(pJson, "cpuModel", tmp); + tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); + } else { + tjsonAddDoubleToObject(pJson, "numOfCpu", taosGetCpuCores()); } - // read something to avoid nginx error 499 - if (taosReadSocket(fd, buf, 10) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - mError("failed to receive response since %s", terrstr()); - goto SEND_OVER; + uint64_t memoryKB = 0; + if (taosGetTotalSysMemoryKB(&memoryKB)) { + snprintf(tmp, sizeof(tmp), "%" PRIu64 " kB", memoryKB); + tjsonAddStringToObject(pJson, "memory", tmp); } - mInfo("send telemetry to %s:%d, len:%d content: %s", TELEMETRY_SERVER, TELEMETRY_PORT, contLen, pCont); - code = 0; + tjsonAddStringToObject(pJson, "version", version); + tjsonAddStringToObject(pJson, "buildInfo", buildinfo); + tjsonAddStringToObject(pJson, "gitInfo", gitinfo); + tjsonAddStringToObject(pJson, "email", pMgmt->email); -SEND_OVER: - tbufCloseWriter(&bw); - taosCloseSocket(fd); + mndBuildRuntimeInfo(pMnode, pJson); - if (code != 0) { - mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); - } + char* pCont = tjsonToString(pJson); + tjsonDelete(pJson); + return pCont; } static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { @@ -247,33 +88,23 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { if (!pMgmt->enable) return 0; taosWLockLatch(&pMgmt->lock); - mndSendTelemetryReport(pMnode); + char* pCont = mndBuildTelemetryReport(pMnode); + if (pCont != NULL) { + taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont)); + free(pCont); + } taosWUnLockLatch(&pMgmt->lock); return 0; } -static void mndGetEmail(SMnode* pMnode, char* filepath) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - - TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); - if (pFile == NULL) { - return; - } - - if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { - mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); - } - - taosCloseFile(&pFile); -} - int32_t mndInitTelem(SMnode* pMnode) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - pMgmt->enable = tsEnableTelemetryReporting; - taosInitRWLatch(&pMgmt->lock); - mndGetEmail(pMnode, "/usr/local/taos/email"); + taosInitRWLatch(&pMgmt->lock); + pMgmt->enable = tsEnableTelemetryReporting; + taosGetEmail(pMgmt->email, sizeof(pMgmt->email)); mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); + mDebug("mnode telemetry is initialized"); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index d2318009d59202b0f3eaa5f7ca727fcf78b72802..9822550ee536d093864b55522deb6e2bb7aa4996 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -72,7 +72,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { if (pRaw == NULL) goto TOPIC_ENCODE_OVER; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, TOPIC_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TOPIC_FNAME_LEN, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->createTime, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER); @@ -121,7 +121,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { int32_t len; int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, TOPIC_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TOPIC_FNAME_LEN, TOPIC_DECODE_OVER); SDB_GET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->createTime, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER); @@ -206,7 +206,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { SName name = {0}; tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - char db[TSDB_TABLE_FNAME_LEN] = {0}; + char db[TSDB_TOPIC_FNAME_LEN] = {0}; tNameGetFullDbName(&name, db); return mndAcquireDb(pMnode, db); @@ -223,7 +223,7 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq pDrop->head.contLen = htonl(contLen); pDrop->head.vgId = htonl(pVgroup->vgId); - memcpy(pDrop->name, pTopic->name, TSDB_TABLE_FNAME_LEN); + memcpy(pDrop->name, pTopic->name, TSDB_TOPIC_FNAME_LEN); pDrop->tuid = htobe64(pTopic->uid); return pDrop; @@ -343,6 +343,7 @@ CREATE_TOPIC_OVER: } static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pReq, SMqTopicObj *pTopic) { + // TODO: cannot drop when subscribed STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_TOPIC, &pReq->rpcMsg); if (pTrans == NULL) { mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); @@ -408,75 +409,34 @@ static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) { return 0; } -static int32_t mndProcessTopicMetaReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - STableInfoReq infoReq = {0}; - - if (tSerializeSTableInfoReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &infoReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - mDebug("topic:%s, start to retrieve meta", infoReq.tbName); - #if 0 - SDbObj *pDb = mndAcquireDbByTopic(pMnode, pInfo->tableFname); +static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { + SSdb *pSdb = pMnode->pSdb; + SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("topic:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr()); return -1; } - STopicObj *pTopic = mndAcquireTopic(pMnode, pInfo->tableFname); - if (pTopic == NULL) { - mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_MND_INVALID_TOPIC; - mError("topic:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); - return -1; - } + int32_t numOfTopics = 0; + void *pIter = NULL; + while (1) { + SMqTopicObj *pTopic = NULL; + pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); + if (pIter == NULL) break; - taosRLockLatch(&pTopic->lock); - int32_t totalCols = pTopic->numOfColumns + pTopic->numOfTags; - int32_t contLen = sizeof(STableMetaRsp) + totalCols * sizeof(SSchema); + if (pTopic->dbUid == pDb->uid) { + numOfTopics++; + } - STableMetaRsp *pMeta = rpcMallocCont(contLen); - if (pMeta == NULL) { - taosRUnLockLatch(&pTopic->lock); - mndReleaseDb(pMnode, pDb); - mndReleaseTopic(pMnode, pTopic); - terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("topic:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); - return -1; + sdbRelease(pSdb, pTopic); } - memcpy(pMeta->topicFname, pTopic->name, TSDB_TABLE_FNAME_LEN); - pMeta->numOfTags = htonl(pTopic->numOfTags); - pMeta->numOfColumns = htonl(pTopic->numOfColumns); - pMeta->precision = pDb->cfg.precision; - pMeta->tableType = TSDB_SUPER_TABLE; - pMeta->update = pDb->cfg.update; - pMeta->sversion = htonl(pTopic->version); - pMeta->tuid = htonl(pTopic->uid); - - for (int32_t i = 0; i < totalCols; ++i) { - SSchema *pSchema = &pMeta->pSchemas[i]; - SSchema *pSrcSchema = &pTopic->pSchema[i]; - memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); - pSchema->type = pSrcSchema->type; - pSchema->colId = htonl(pSrcSchema->colId); - pSchema->bytes = htonl(pSrcSchema->bytes); - } - taosRUnLockLatch(&pTopic->lock); + *pNumOfTopics = numOfTopics; mndReleaseDb(pMnode, pDb); - mndReleaseTopic(pMnode, pTopic); - - pReq->pCont = pMeta; - pReq->contLen = contLen; - - mDebug("topic:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pTopic->numOfColumns, pTopic->numOfTags); -#endif return 0; } +#endif static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { SSdb *pSdb = pMnode->pSdb; @@ -571,7 +531,7 @@ static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, in if (pTopic->dbUid != pDb->uid) { if (strncmp(pTopic->name, prefix, prefixLen) != 0) { - mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pTopic->name, pDb->name, pDb->uid); + mError("Inconsistent topic data, name:%s, db:%s, dbUid:%" PRIu64, pTopic->name, pDb->name, pDb->uid); } sdbRelease(pSdb, pTopic); @@ -580,8 +540,8 @@ static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, in cols = 0; - char topicName[TSDB_TABLE_NAME_LEN] = {0}; - tstrncpy(topicName, pTopic->name + prefixLen, TSDB_TABLE_NAME_LEN); + char topicName[TSDB_TOPIC_NAME_LEN] = {0}; + tstrncpy(topicName, pTopic->name + prefixLen, TSDB_TOPIC_NAME_LEN); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; STR_TO_VARSTR(pWrite, topicName); cols++; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index ab5d0d794b6f0416056f418dbc83d104f7362d69..90d66a80fdc943837b6e990164552de43f4b1272 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -489,7 +489,7 @@ PROCESS_RPC_END: if (code == TSDB_CODE_APP_NOT_READY) { mndSendRedirectRsp(pMnode, &pMsg->rpcMsg); } else if (code != 0) { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .code = code}; + SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont, .code = code}; rpcSendResponse(&rpcRsp); } else { SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .contLen = pMsg->contLen, .pCont = pMsg->pCont}; diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index ffab15f39446617cb33ab7d878a1a9e3fdca1f26..36626514ec1c3c7925295f2babb8d479c03f8467 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -51,7 +51,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl void tqClose(STQ*); // required by vnode -int tqPushMsg(STQ*, void* msg, int64_t version); +int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version); int tqCommit(STQ*); int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index e87de10912f6f27b8b62d73dd77d4a2a70c74a98..30a83ca63446dbcf11448d5f06728353d6b97010 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -79,19 +79,19 @@ extern int32_t tqDebugFlag; // 4096 - 4080 #define TQ_IDX_PAGE_HEAD_SIZE 16 -#define TQ_ACTION_CONST 0 -#define TQ_ACTION_INUSE 1 +#define TQ_ACTION_CONST 0 +#define TQ_ACTION_INUSE 1 #define TQ_ACTION_INUSE_CONT 2 -#define TQ_ACTION_INTXN 3 +#define TQ_ACTION_INTXN 3 #define TQ_SVER 0 // TODO: inplace mode is not implemented #define TQ_UPDATE_INPLACE 0 -#define TQ_UPDATE_APPEND 1 +#define TQ_UPDATE_APPEND 1 #define TQ_DUP_INTXN_REWRITE 0 -#define TQ_DUP_INTXN_REJECT 2 +#define TQ_DUP_INTXN_REJECT 2 static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } @@ -160,7 +160,7 @@ struct STQ { STqMemRef tqMemRef; STqMetaStore* tqMeta; SWal* pWal; - SMeta* pMeta; + SMeta* pVnodeMeta; }; typedef struct { @@ -190,9 +190,6 @@ typedef struct { char* logicalPlan; char* physicalPlan; char* qmsg; - int64_t persistedOffset; - int64_t committedOffset; - int64_t currentOffset; STqBuffer buffer; SWalReadHandle* pReadhandle; } STqTopic; @@ -201,7 +198,7 @@ typedef struct { int64_t consumerId; int64_t epoch; char cgroup[TSDB_TOPIC_FNAME_LEN]; - SArray* topics; // SArray + SArray* topics; // SArray } STqConsumer; int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ac9dde35974227538b4c8aee7ba5d15ecfeee9bb..aa198d080699a814171a4984cd8d8228671eeecd 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -42,7 +42,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl pTq->path = strdup(path); pTq->tqConfig = tqConfig; pTq->pWal = pWal; - pTq->pMeta = pMeta; + pTq->pVnodeMeta = pMeta; #if 0 pTq->tqMemRef.pAllocatorFactory = allocFac; pTq->tqMemRef.pAllocator = allocFac->create(allocFac); @@ -71,9 +71,11 @@ void tqClose(STQ* pTq) { // TODO } -int tqPushMsg(STQ* pTq, void* p, int64_t version) { - // add reference - // judge and launch new query +int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { + // TODO: add reference + // if handle waiting, launch query and response to consumer + // + // if no waiting handle, return return 0; } @@ -101,9 +103,9 @@ static FORCE_INLINE int32_t tEncodeSTqTopic(void** buf, const STqTopic* pTopic) /*tlen += taosEncodeString(buf, pTopic->logicalPlan);*/ /*tlen += taosEncodeString(buf, pTopic->physicalPlan);*/ tlen += taosEncodeString(buf, pTopic->qmsg); - tlen += taosEncodeFixedI64(buf, pTopic->persistedOffset); - tlen += taosEncodeFixedI64(buf, pTopic->committedOffset); - tlen += taosEncodeFixedI64(buf, pTopic->currentOffset); + /*tlen += taosEncodeFixedI64(buf, pTopic->persistedOffset);*/ + /*tlen += taosEncodeFixedI64(buf, pTopic->committedOffset);*/ + /*tlen += taosEncodeFixedI64(buf, pTopic->currentOffset);*/ return tlen; } @@ -113,9 +115,9 @@ static FORCE_INLINE const void* tDecodeSTqTopic(const void* buf, STqTopic* pTopi /*buf = taosDecodeString(buf, &pTopic->logicalPlan);*/ /*buf = taosDecodeString(buf, &pTopic->physicalPlan);*/ buf = taosDecodeString(buf, &pTopic->qmsg); - buf = taosDecodeFixedI64(buf, &pTopic->persistedOffset); - buf = taosDecodeFixedI64(buf, &pTopic->committedOffset); - buf = taosDecodeFixedI64(buf, &pTopic->currentOffset); + /*buf = taosDecodeFixedI64(buf, &pTopic->persistedOffset);*/ + /*buf = taosDecodeFixedI64(buf, &pTopic->committedOffset);*/ + /*buf = taosDecodeFixedI64(buf, &pTopic->currentOffset);*/ return buf; } @@ -194,8 +196,8 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu } for (int j = 0; j < TQ_BUFFER_SIZE; j++) { pTopic->buffer.output[j].status = 0; - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); - SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pMeta}; + STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); + SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta}; pTopic->buffer.output[j].pReadHandle = pReadHandle; pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); } @@ -218,7 +220,11 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { fetchOffset = pReq->currentOffset + 1; } - SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; + SMqConsumeRsp rsp = { + .consumerId = consumerId, + .numOfTopics = 0, + .pBlockData = NULL, + }; STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); if (pConsumer == NULL) { @@ -243,7 +249,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and - // rsponse to user + // response to user break; } pHead = pTopic->pReadhandle->pHead; @@ -268,19 +274,20 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { taosArrayPush(pRes, pDataBlock); rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; rsp.rspOffset = fetchOffset; - pTopic->currentOffset = fetchOffset; rsp.numOfTopics = 1; rsp.pBlockData = pRes; - int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp); + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp); void* buf = rpcMallocCont(tlen); if (buf == NULL) { pMsg->code = -1; return -1; } + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + ((SMqRspHead*)buf)->epoch = pReq->epoch; - void* abuf = buf; + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqConsumeRsp(&abuf, &rsp); taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock); pMsg->pCont = buf; @@ -295,14 +302,16 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } } - int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp); + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp); void* buf = rpcMallocCont(tlen); if (buf == NULL) { pMsg->code = -1; return -1; } + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + ((SMqRspHead*)buf)->epoch = pReq->epoch; - void* abuf = buf; + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqConsumeRsp(&abuf, &rsp); rsp.pBlockData = NULL; pMsg->pCont = buf; @@ -312,158 +321,6 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { return 0; } -#if 0 -int32_t tqProcessConsumeReqV0(STQ* pTq, SRpcMsg* pMsg) { - SMqConsumeReq* pReq = pMsg->pCont; - int64_t reqId = pReq->reqId; - int64_t consumerId = pReq->consumerId; - int64_t fetchOffset = pReq->offset; - int64_t blockingTime = pReq->blockingTime; - - SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; - - /*printf("vg %d get consume req\n", pReq->head.vgId);*/ - - STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); - if (pConsumer == NULL) { - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = -1; - rpcSendResponse(pMsg); - return 0; - } - int sz = taosArrayGetSize(pConsumer->topics); - - for (int i = 0; i < sz; i++) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); - // TODO: support multiple topic in one req - if (strcmp(pTopic->topicName, pReq->topic) != 0) { - ASSERT(false); - continue; - } - - if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) { - pTopic->committedOffset = pReq->offset; - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = 0; - rpcSendResponse(pMsg); - return 0; - } - - if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { - pTopic->committedOffset = pReq->offset - 1; - } - - rsp.committedOffset = pTopic->committedOffset; - rsp.reqOffset = pReq->offset; - rsp.skipLogNum = 0; - - if (fetchOffset <= pTopic->committedOffset) { - fetchOffset = pTopic->committedOffset + 1; - } - /*printf("vg %d fetch Offset %ld\n", pReq->head.vgId, fetchOffset);*/ - int8_t pos; - int8_t skip = 0; - SWalHead* pHead; - while (1) { - pos = fetchOffset % TQ_BUFFER_SIZE; - skip = atomic_val_compare_exchange_8(&pTopic->buffer.output[pos].status, 0, 1); - if (skip == 1) { - // do nothing - break; - } - if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { - printf("read offset %ld\n", fetchOffset); - // check err - atomic_store_8(&pTopic->buffer.output[pos].status, 0); - skip = 1; - break; - } - // read until find TDMT_VND_SUBMIT - pHead = pTopic->pReadhandle->pHead; - if (pHead->head.msgType == TDMT_VND_SUBMIT) { - } - rsp.skipLogNum++; - if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { - printf("read offset %ld\n", fetchOffset); - atomic_store_8(&pTopic->buffer.output[pos].status, 0); - skip = 1; - break; - } - atomic_store_8(&pTopic->buffer.output[pos].status, 0); - fetchOffset++; - } - if (skip == 1) continue; - SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body; - qTaskInfo_t task = pTopic->buffer.output[pos].task; - - printf("current fetch offset %ld\n", fetchOffset); - qSetStreamInput(task, pCont); - - // SArray - SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - while (1) { - SSDataBlock* pDataBlock; - uint64_t ts; - if (qExecTask(task, &pDataBlock, &ts) < 0) { - break; - } - if (pDataBlock != NULL) { - taosArrayPush(pRes, pDataBlock); - } else { - break; - } - } - // TODO copy - rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; - rsp.rspOffset = fetchOffset; - pTopic->currentOffset = fetchOffset; - - atomic_store_8(&pTopic->buffer.output[pos].status, 0); - - if (taosArrayGetSize(pRes) == 0) { - taosArrayDestroy(pRes); - fetchOffset++; - continue; - } else { - rsp.numOfTopics++; - } - - rsp.pBlockData = pRes; - -#if 0 - pTopic->buffer.output[pos].dst = pRes; - if (pTopic->buffer.firstOffset == -1 || pReq->offset < pTopic->buffer.firstOffset) { - pTopic->buffer.firstOffset = pReq->offset; - } - if (pTopic->buffer.lastOffset == -1 || pReq->offset > pTopic->buffer.lastOffset) { - pTopic->buffer.lastOffset = pReq->offset; - } -#endif - } - int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - pMsg->code = -1; - return -1; - } - void* abuf = buf; - tEncodeSMqConsumeRsp(&abuf, &rsp); - - if (rsp.pBlockData) { - taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock); - rsp.pBlockData = NULL; - } - - pMsg->pCont = buf; - pMsg->contLen = tlen; - pMsg->code = 0; - rpcSendResponse(pMsg); - return 0; -} -#endif - int32_t tqProcessRebReq(STQ* pTq, char* msg) { SMqMVRebReq req = {0}; tDecodeSMqMVRebReq(msg, &req); @@ -505,8 +362,8 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { pTopic->logicalPlan = req.logicalPlan; pTopic->physicalPlan = req.physicalPlan; pTopic->qmsg = req.qmsg; - pTopic->committedOffset = -1; - pTopic->currentOffset = -1; + /*pTopic->committedOffset = -1;*/ + /*pTopic->currentOffset = -1;*/ pTopic->buffer.firstOffset = -1; pTopic->buffer.lastOffset = -1; @@ -516,8 +373,8 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { } for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); - SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pMeta}; + STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); + SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta}; pTopic->buffer.output[i].pReadHandle = pReadHandle; pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); } diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index c3947da4591de26b9a4efbc6cb5b23204e8fbb49..81eb09f48fcc6e00bf2bce5f2ceebe0bded4e575 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -59,7 +59,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // todo: change the interface here int64_t ver; taosDecodeFixedI64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver); - if (tqPushMsg(pVnode->pTq, ptr, ver) < 0) { + if (tqPushMsg(pVnode->pTq, ptr, pMsg->msgType, ver) < 0) { // TODO: handle error } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 2b6caa2f8fb91f791f2ab30989398e9e12bf754e..f0ea51c2f9e28b6a9999d1d8d75bbe8821f5483e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -166,6 +166,28 @@ void ctgDbgShowDBCache(SHashObj *dbHash) { taosHashGetKey(dbCache, (void **)&dbFName, &len); CTG_CACHE_DEBUG("** %dth db [%.*s][%"PRIx64"] **", i, (int32_t)len, dbFName, dbCache->dbId); + + CTG_CACHE_DEBUG("deleted: %d", dbCache->deleted); + if (dbCache->vgInfo) { + CTG_CACHE_DEBUG("vgVersion: %d", dbCache->vgInfo->vgVersion); + CTG_CACHE_DEBUG("hashMethod: %d", dbCache->vgInfo->hashMethod); + if (dbCache->vgInfo->vgHash) { + CTG_CACHE_DEBUG("vgNum: %d", taosHashGetSize(dbCache->vgInfo->vgHash)); + //TODO + } else { + CTG_CACHE_DEBUG("vgHash: %p", dbCache->vgInfo->vgHash); + } + } else { + CTG_CACHE_DEBUG("vgInfo: %p", dbCache->vgInfo); + } + + if (dbCache->tbCache.metaCache) { + CTG_CACHE_DEBUG("metaNum: %d", taosHashGetSize(dbCache->tbCache.metaCache)); + } + + if (dbCache->tbCache.stbCache) { + CTG_CACHE_DEBUG("stbNum: %d", taosHashGetSize(dbCache->tbCache.stbCache)); + } pIter = taosHashIterate(dbHash, pIter); } @@ -242,6 +264,34 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { } +int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; + SCtgRemoveDBMsg *msg = malloc(sizeof(SCtgRemoveDBMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(action.data); + CTG_RET(code); +} + + void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { @@ -452,12 +502,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtE rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { - if (CTG_DB_NOT_EXIST(rpcRsp.code)) { - ctgDebug("db not exist in mnode, dbFName:%s", input->db); - return rpcRsp.code; - } - - ctgError("error rsp for use db, code:%s, db:%s", tstrerror(rpcRsp.code), input->db); + ctgError("error rsp for use db, error:%s, db:%s", tstrerror(rpcRsp.code), input->db); CTG_ERR_RET(rpcRsp.code); } @@ -1365,20 +1410,33 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, bool forceUpdate, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { bool inCache = false; int32_t code = 0; - if (!forceUpdate) { - CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); - if (inCache) { - return TSDB_CODE_SUCCESS; - } + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); + + if (inCache && !forceUpdate) { + return TSDB_CODE_SUCCESS; } SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); - input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + if (inCache) { + input.dbId = (*dbCache)->dbId; + input.vgVersion = (*dbCache)->vgInfo->vgVersion; + } else { + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + } - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut)); + code = ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut); + if (code) { + if (CTG_DB_NOT_EXIST(code) && input.vgVersion > CTG_DEFAULT_INVALID_VERSION) { + ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId); + ctgPushRmDBMsgInQueue(pCtg, input.db, input.dbId); + } + + CTG_ERR_RET(code); + } CTG_ERR_JRET(ctgCloneVgInfo(DbOut.dbVgroup, pInfo)); @@ -1772,7 +1830,6 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { } - void* ctgUpdateThreadFunc(void* param) { setThreadName("catalog"); @@ -1964,10 +2021,10 @@ void catalogFreeHandle(SCatalog* pCtg) { ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); } -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version) { +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == dbFName || NULL == version) { + if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1994,6 +2051,7 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers } *version = dbCache->vgInfo->vgVersion; + *dbId = dbCache->dbId; ctgReleaseVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); @@ -2099,29 +2157,12 @@ int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; - SCtgRemoveDBMsg *msg = malloc(sizeof(SCtgRemoveDBMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); - CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - msg->dbId = dbId; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(&action)); - - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_ERR_JRET(ctgPushRmDBMsgInQueue(pCtg, dbFName, dbId)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); _return: - tfree(action.data); - CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index f3aecadd3fef9fb3231ef720e23954058861ca67..b7432429f4cf59b24a06b366b5db01f29211f78d 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -128,6 +128,7 @@ void ctgTestInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); ctgDbgEnableDebug("api"); @@ -1498,7 +1499,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM)) { usleep(10000); } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index e23c9ebe9d21fb8842b313f3ad886d2f813dba5d..a125906faf961fddd6f5be0a3abbbd943c0ab1f0 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -157,7 +157,7 @@ static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode return (SNode*)pDst; } -SNode* nodesCloneNode(const SNode* pNode) { +SNodeptr nodesCloneNode(const SNodeptr pNode) { if (NULL == pNode) { return NULL; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index fd54537d6562496e6beffaf3d61f282abedc1150..dd0eac00c51947f2024e8f2d19031124ee86580c 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1351,7 +1351,7 @@ static int32_t jsonToNodeObject(const SJson* pJson, const char* pName, SNode** p return makeNodeByJson(pJsonNode, pNode); } -int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen) { +int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen) { if (NULL == pNode || NULL == pStr || NULL == pLen) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index fb2463d350841c72b162cd73fe817b63bbfafda8..4185754355ddfd98fc9c7243ab961ed727a8d4d4 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -95,7 +95,7 @@ static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { return true; } -bool nodesEqualNode(const SNode* a, const SNode* b) { +bool nodesEqualNode(const SNodeptr a, const SNodeptr b) { if (a == b) { return true; } diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 5c81cce94aa66f8b082e686c509fef483eff13e2..e3c2dff1208b427db0df79dfe1eafd2fb7682c2b 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -129,7 +129,7 @@ static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalke return DEAL_RES_CONTINUE; } -void nodesWalkNode(SNode* pNode, FNodeWalker walker, void* pContext) { +void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); } @@ -137,7 +137,7 @@ void nodesWalkList(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkNodePostOrder(SNode* pNode, FNodeWalker walker, void* pContext) { +void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 70dcab745f1d49659a9d753b306d61f69c54cdc7..ce68816f51ab5fc05e37d6ca68cb7a4de6925d84 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -29,7 +29,7 @@ static SNode* makeNode(ENodeType type, size_t size) { return p; } -SNode* nodesMakeNode(ENodeType type) { +SNodeptr nodesMakeNode(ENodeType type) { switch (type) { case QUERY_NODE_COLUMN: return makeNode(type, sizeof(SColumnNode)); @@ -136,11 +136,11 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -void nodesDestroyNode(SNode* pNode) { +void nodesDestroyNode(SNodeptr pNode) { if (NULL == pNode) { return; } - nodesRewriteNodePostOrder(&pNode, destroyNode, NULL); + nodesRewriteNodePostOrder((SNode**)&pNode, destroyNode, NULL); } SNodeList* nodesMakeList() { @@ -151,7 +151,7 @@ SNodeList* nodesMakeList() { return p; } -int32_t nodesListAppend(SNodeList* pList, SNode* pNode) { +int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode) { if (NULL == pList || NULL == pNode) { return TSDB_CODE_SUCCESS; } @@ -206,7 +206,7 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { return pNext; } -SNode* nodesListGetNode(SNodeList* pList, int32_t index) { +SNodeptr nodesListGetNode(SNodeList* pList, int32_t index) { SNode* node; FOREACH(node, pList) { if (0 == index--) { diff --git a/source/libs/nodes/test/nodesTest.cpp b/source/libs/nodes/test/nodesTest.cpp index c116faf4cee3705c9cfdc51f8508411897d65eb4..fd4a9e8c20c3c13ea47b327559748c60a37f04d0 100644 --- a/source/libs/nodes/test/nodesTest.cpp +++ b/source/libs/nodes/test/nodesTest.cpp @@ -36,15 +36,15 @@ static EDealRes rewriterTest(SNode** pNode, void* pContext) { } TEST(NodesTest, traverseTest) { - SNode* pRoot = nodesMakeNode(QUERY_NODE_OPERATOR); + SNode* pRoot = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR); SOperatorNode* pOp = (SOperatorNode*)pRoot; SOperatorNode* pLeft = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); - pLeft->pLeft = nodesMakeNode(QUERY_NODE_VALUE); + pLeft->pLeft = (SNode*)nodesMakeNode(QUERY_NODE_VALUE); ((SValueNode*)(pLeft->pLeft))->literal = strdup("10"); - pLeft->pRight = nodesMakeNode(QUERY_NODE_VALUE); + pLeft->pRight = (SNode*)nodesMakeNode(QUERY_NODE_VALUE); ((SValueNode*)(pLeft->pRight))->literal = strdup("5"); pOp->pLeft = (SNode*)pLeft; - pOp->pRight = nodesMakeNode(QUERY_NODE_VALUE); + pOp->pRight = (SNode*)nodesMakeNode(QUERY_NODE_VALUE); ((SValueNode*)(pOp->pRight))->literal = strdup("3"); EXPECT_EQ(nodeType(pRoot), QUERY_NODE_OPERATOR); diff --git a/source/libs/parser/test/parserTest.cpp b/source/libs/parser/test/parserTest.cpp index c714c0b8f5a6f9a6e040cb5587c71f4a4406a23f..76edf60338a42ccb98ecf0bacbcb1756e305b67f 100644 --- a/source/libs/parser/test/parserTest.cpp +++ b/source/libs/parser/test/parserTest.cpp @@ -207,7 +207,7 @@ private: } void groupingSetToStr(SGroupingSetNode* pGroup, string& str) { - nodeToStr(nodesListGetNode(pGroup->pParameterList, 0), str, false); + nodeToStr((SNode*)nodesListGetNode(pGroup->pParameterList, 0), str, false); } void orderByExprToStr(SOrderByExprNode* pOrderBy, string& str) { diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 6d2205f72b8ca150322d8a898eb37fb6ad751f4d..050298501d0bcd79614ab0affb14d9d4b4d71296 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -952,6 +952,17 @@ static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLog return pSubplan; } +static int32_t strictListAppend(SNodeList* pList, SNodeptr pNode) { + if (NULL == pNode) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t code = nodesListAppend(pList, pNode); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNode); + } + return code; +} + static SQueryLogicPlan* createRawQueryLogicPlan(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode) { SQueryLogicPlan* pLogicPlan = (SQueryLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN); CHECK_ALLOC(pLogicPlan, NULL); @@ -959,18 +970,12 @@ static SQueryLogicPlan* createRawQueryLogicPlan(SPhysiPlanContext* pCxt, SLogicN CHECK_ALLOC(pLogicPlan->pSubplans, pLogicPlan); SNodeListNode* pTopSubplans = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); CHECK_ALLOC(pTopSubplans, pLogicPlan); - if (TSDB_CODE_SUCCESS != nodesListAppend(pLogicPlan->pSubplans, (SNode*)pTopSubplans)) { - nodesDestroyNode((SNode*)pTopSubplans); - return pLogicPlan; - } + CHECK_CODE(strictListAppend(pLogicPlan->pSubplans, pTopSubplans), pLogicPlan); pTopSubplans->pNodeList = nodesMakeList(); CHECK_ALLOC(pTopSubplans->pNodeList, pLogicPlan); SSubLogicPlan* pSubplan = (SSubLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); CHECK_ALLOC(pSubplan, pLogicPlan); - if (TSDB_CODE_SUCCESS != nodesListAppend(pTopSubplans->pNodeList, (SNode*)pSubplan)) { - nodesDestroyNode((SNode*)pSubplan); - return pLogicPlan; - } + CHECK_CODE(strictListAppend(pTopSubplans->pNodeList, pSubplan), pLogicPlan); pSubplan->pNode = (SLogicNode*)nodesCloneNode((SNode*)pLogicNode); CHECK_ALLOC(pSubplan->pNode, pLogicPlan); return pLogicPlan; @@ -999,18 +1004,11 @@ static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogicPl FOREACH(pNode, pLogicPlan->pSubplans) { SNodeListNode* pLevelSubplans = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); CHECK_ALLOC(pLevelSubplans, TSDB_CODE_OUT_OF_MEMORY); - if (TSDB_CODE_SUCCESS != nodesListAppend(pQueryPlan->pSubplans, (SNode*)pLevelSubplans)) { - nodesDestroyNode((SNode*)pLevelSubplans); - return TSDB_CODE_OUT_OF_MEMORY; - } + CHECK_CODE(strictListAppend(pQueryPlan->pSubplans, pLevelSubplans), TSDB_CODE_OUT_OF_MEMORY); SNode* pLogicSubplan; FOREACH(pLogicSubplan, ((SNodeListNode*)pNode)->pNodeList) { - SSubplan* pSubplan = createPhysiSubplan(pCxt, (SSubLogicPlan*)pLogicSubplan); - CHECK_ALLOC(pSubplan, TSDB_CODE_OUT_OF_MEMORY); - if (TSDB_CODE_SUCCESS != nodesListAppend(pLevelSubplans->pNodeList, (SNode*)pSubplan)) { - nodesDestroyNode((SNode*)pSubplan); - return TSDB_CODE_OUT_OF_MEMORY; - } + CHECK_CODE(strictListAppend(pLevelSubplans->pNodeList, + createPhysiSubplan(pCxt, (SSubLogicPlan*)pLogicSubplan)), TSDB_CODE_OUT_OF_MEMORY); ++(pQueryPlan->numOfSubplans); } } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 8bc29656d41300a9db7c7ba6d22a22c83c72001d..2a52e01dc17d7a92b513834831eff6c72f055e0a 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -24,6 +24,32 @@ int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; int32_t (*queryProcessMsgRsp[TDMT_MAX])(void *output, char *msg, int32_t msgSize) = {0}; +int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { + memcpy(pOut->db, usedbRsp->db, TSDB_DB_FNAME_LEN); + pOut->dbId = usedbRsp->uid; + pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); + if (NULL == pOut->dbVgroup) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pOut->dbVgroup->vgVersion = usedbRsp->vgVersion; + pOut->dbVgroup->hashMethod = usedbRsp->hashMethod; + pOut->dbVgroup->vgHash = + taosHashInit(usedbRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == pOut->dbVgroup->vgHash) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < usedbRsp->vgNum; ++i) { + SVgroupInfo *pVgInfo = taosArrayGet(usedbRsp->pVgroupInfos, i); + if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { SBuildTableMetaInput *pInput = input; if (NULL == input || NULL == msg || NULL == msgLen) { @@ -57,6 +83,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms strncpy(usedbReq.db, pInput->db, sizeof(usedbReq.db)); usedbReq.db[sizeof(usedbReq.db) - 1] = 0; usedbReq.vgVersion = pInput->vgVersion; + usedbReq.dbId = pInput->dbId; int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); void *pBuf = rpcMallocCont(bufLen); @@ -90,35 +117,10 @@ int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { goto PROCESS_USEDB_OVER; } - memcpy(pOut->db, usedbRsp.db, TSDB_DB_FNAME_LEN); - pOut->dbId = usedbRsp.uid; - pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); - if (NULL == pOut->dbVgroup) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto PROCESS_USEDB_OVER; - } - - pOut->dbVgroup->vgVersion = usedbRsp.vgVersion; - pOut->dbVgroup->hashMethod = usedbRsp.hashMethod; - pOut->dbVgroup->vgHash = - taosHashInit(usedbRsp.vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == pOut->dbVgroup->vgHash) { - tfree(pOut->dbVgroup); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto PROCESS_USEDB_OVER; - } - - for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { - SVgroupInfo *pVgInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); - if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto PROCESS_USEDB_OVER; - } - } - - code = 0; + code = queryBuildUseDbOutput(pOut, &usedbRsp); PROCESS_USEDB_OVER: + if (code != 0) { if (pOut) { if (pOut->dbVgroup) taosHashCleanup(pOut->dbVgroup->vgHash); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index fafc1ea42e860a8c410d75c1e58738f3c1e9d79e..0184923638dcec2dc9ef683f79f8d6459757f96d 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -60,7 +60,7 @@ void flttInitLogFile() { void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { - SNode *node = nodesMakeNode(QUERY_NODE_VALUE); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_VALUE); SValueNode *vnode = (SValueNode *)node; vnode->node.resType.type = dataType; @@ -79,7 +79,7 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { static uint64_t dbidx = 0; - SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; rnode->node.resType.bytes = dataBytes; @@ -168,7 +168,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } void flttMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { - SNode *node = nodesMakeNode(QUERY_NODE_OPERATOR); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR); SOperatorNode *onode = (SOperatorNode *)node; onode->node.resType.type = resType; onode->node.resType.bytes = tDataTypes[resType].bytes; @@ -181,7 +181,7 @@ void flttMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode } void flttMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) { - SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); SLogicConditionNode *onode = (SLogicConditionNode *)node; onode->condType = opType; onode->node.resType.type = TSDB_DATA_TYPE_BOOL; @@ -196,7 +196,7 @@ void flttMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeLi } void flttMakeLogicNodeFromList(SNode **pNode, ELogicConditionType opType, SNodeList *nodeList) { - SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); SLogicConditionNode *onode = (SLogicConditionNode *)node; onode->condType = opType; onode->node.resType.type = TSDB_DATA_TYPE_BOOL; @@ -208,7 +208,7 @@ void flttMakeLogicNodeFromList(SNode **pNode, ELogicConditionType opType, SNodeL } void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { - SNode *node = nodesMakeNode(QUERY_NODE_NODE_LIST); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *lnode = (SNodeListNode *)node; lnode->dataType.type = resType; lnode->pNodeList = list; diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index b9aef99088bef2fefb98a7447cd77c6945c1ba76..5e729543215d08bf8e41acfad499ee499d9ea4b2 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -93,7 +93,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s } void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { - SNode *node = nodesMakeNode(QUERY_NODE_VALUE); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_VALUE); SValueNode *vnode = (SValueNode *)node; vnode->node.resType.type = dataType; @@ -110,7 +110,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { } void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { - SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; rnode->node.resType.bytes = dataBytes; @@ -189,7 +189,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { - SNode *node = nodesMakeNode(QUERY_NODE_OPERATOR); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR); SOperatorNode *onode = (SOperatorNode *)node; onode->node.resType.type = resType; onode->node.resType.bytes = tDataTypes[resType].bytes; @@ -203,7 +203,7 @@ void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode void scltMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { - SNode *node = nodesMakeNode(QUERY_NODE_NODE_LIST); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *lnode = (SNodeListNode *)node; lnode->dataType.type = resType; lnode->pNodeList = list; @@ -213,7 +213,7 @@ void scltMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { void scltMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) { - SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); SLogicConditionNode *onode = (SLogicConditionNode *)node; onode->condType = opType; onode->node.resType.type = TSDB_DATA_TYPE_BOOL; @@ -228,7 +228,7 @@ void scltMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeLi } void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNode *snode) { - SNode *node = nodesMakeNode(QUERY_NODE_TARGET); + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_TARGET); STargetNode *onode = (STargetNode *)node; onode->pExpr = snode; onode->dataBlockId = dataBlockId; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index c464073e5faeb6a9a08da886268f469189b2fbf1..91ef97e66be08df3f11e8ea44938005854ff8f96 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -140,4 +140,13 @@ int32_t taosRealPath(char *dirname, int32_t maxlen) { return 0; } +bool taosIsDir(const char *dirname) { + DIR *dir = opendir(dirname); + if (dir != NULL) { + closedir(dir); + return true; + } + return false; +} + #endif diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 2d77df9b435b78c7f237cdc2b132efc1c8473039..4b10a179a955d02fde5fe12816e695c69f9d55a0 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -196,11 +196,11 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { char *mode = NULL; if (tdFileOptions & TD_FILE_APPEND) { mode = (tdFileOptions & TD_FILE_TEXT) ? "at+" : "ab+"; - }else if (tdFileOptions & TD_FILE_TRUNC) { + } else if (tdFileOptions & TD_FILE_TRUNC) { mode = (tdFileOptions & TD_FILE_TEXT) ? "wt+" : "wb+"; - }else if ((tdFileOptions & TD_FILE_READ) && !(tdFileOptions & TD_FILE_WRITE)) { + } else if ((tdFileOptions & TD_FILE_READ) && !(tdFileOptions & TD_FILE_WRITE)) { mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb"; - }else { + } else { mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; } assert(!(tdFileOptions & TD_FILE_EXCL)); @@ -326,7 +326,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { nleft -= nwritten; tbuf += nwritten; } - fsync(pFile->fd); + return count; } @@ -637,7 +637,7 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { } assert(pFile->fp != NULL); - char buffer[MAX_FPRINTFLINE_BUFFER_SIZE] = {0}; + char buffer[MAX_FPRINTFLINE_BUFFER_SIZE] = {0}; va_list ap; va_start(ap, format); vfprintf(pFile->fp, format, ap); @@ -675,11 +675,11 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict__ ptrBuf) { size_t len = 0; return getline(ptrBuf, &len, pFile->fp); } -int32_t taosEOFFile(TdFilePtr pFile) { +int32_t taosEOFFile(TdFilePtr pFile) { if (pFile == NULL) { return 0; } assert(pFile->fp != NULL); - return feof(pFile->fp); -} \ No newline at end of file + return feof(pFile->fp); +} diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index cf9c557f5ef5b36a23de42adf9b1a54e75a27f4c..163fad803fc11487101c3d106fe69343b4154c47 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -888,4 +888,99 @@ SysNameInfo taosGetSysNameInfo() { return info; } -#endif \ No newline at end of file +bool taosGetEmail(char *email, int32_t maxLen) { + const char *filepath = "/usr/local/taos/email"; + + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) return false; + + if (taosReadFile(pFile, (void *)email, maxLen) < 0) { + taosCloseFile(&pFile); + return false; + } + + taosCloseFile(&pFile); + return true; +} + +bool taosGetOsReleaseName(char *releaseName, int32_t maxLen) { + char *line = NULL; + size_t size = 0; + bool ret = false; + + TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return false; + + while ((size = taosGetLineFile(pFile, &line)) != -1) { + line[size - 1] = '\0'; + if (strncmp(line, "PRETTY_NAME", 11) == 0) { + const char *p = strchr(line, '=') + 1; + if (*p == '"') { + p++; + line[size - 2] = 0; + } + tstrncpy(releaseName, p, maxLen); + ret = true; + break; + } + } + + if (line != NULL) free(line); + taosCloseFile(&pFile); + return ret; +} + +bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { + char *line = NULL; + size_t size = 0; + int32_t done = 0; + bool ret = false; + + TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return false; + + while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { + line[size - 1] = '\0'; + if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { + const char *v = strchr(line, ':') + 2; + tstrncpy(cpuModel, v, maxLen); + ret = true; + done |= 1; + } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { + const char *v = strchr(line, ':') + 2; + *numOfCores = atoi(v); + done |= 2; + } + } + + if (line != NULL) free(line); + taosCloseFile(&pFile); + + return ret; +} + +bool taosGetTotalSysMemoryKB(uint64_t *kb) { + char *line = NULL; + size_t size = 0; + bool ret = false; + + TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return false; + + while ((size = taosGetLineFile(pFile, &line)) != -1) { + line[size - 1] = '\0'; + if (strncmp(line, "MemTotal", 8) == 0) { + const char *p = strchr(line, ':') + 1; + while (*p == ' ') p++; + ret = true; + *kb = atoll(p); + break; + } + } + + if (line != NULL) free(line); + taosCloseFile(&pFile); + return ret; +} + +#endif diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 8330c10ec703b7c2484167ff5cba7d46b9eabe85..0f22d15cd46c1269379545f60f745cc80af0d131 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -60,7 +60,7 @@ int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr) { } } -int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs) { +int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs) { int32_t size = taosArrayGetSize(pArgs); for (int32_t i = 0; i < size; ++i) { SConfigPair *pPair = taosArrayGet(pArgs, i); @@ -145,10 +145,12 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } +#if 0 if (taosMkDir(fullDir) != 0) { uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); return -1; } +#endif cfgFreeItem(pItem); pItem->str = strdup(fullDir); @@ -608,7 +610,10 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { int32_t olen, vlen, vlen2, vlen3; ssize_t _bytes = 0; - // FILE *fp = fopen(filepath, "r"); + if (taosIsDir(filepath)) { + return -1; + } + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/util/src/thttp.c b/source/util/src/thttp.c new file mode 100644 index 0000000000000000000000000000000000000000..14c39d3f03f6a59e921de174abe4425bec6f7441 --- /dev/null +++ b/source/util/src/thttp.c @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "thttp.h" +#include "taoserror.h" +#include "tlog.h" + +int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, int32_t contLen) { + int32_t code = -1; + SOCKET fd = 0; + + uint32_t ip = taosGetIpv4FromFqdn(server); + if (ip == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get http server:%s ip since %s", server, terrstr()); + goto SEND_OVER; + } + + fd = taosOpenTcpClientSocket(ip, port, 0); + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to create http socket since %s", terrstr()); + goto SEND_OVER; + } + + char header[4096] = {0}; + int32_t headLen = snprintf(header, sizeof(header), + "POST /report HTTP/1.1\n" + "Host: %s\n" + "Content-Type: application/json\n" + "Content-Length: %d\n\n", + server, contLen); + + if (taosWriteSocket(fd, (void*)header, headLen) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to send http header since %s", terrstr()); + goto SEND_OVER; + } + + if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to send http content since %s", terrstr()); + goto SEND_OVER; + } + + // read something to avoid nginx error 499 + if (taosReadSocket(fd, header, 10) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to receive response since %s", terrstr()); + goto SEND_OVER; + } + + uInfo("send http to %s:%d, len:%d content: %s", server, port, contLen, pCont); + code = 0; + +SEND_OVER: + if (fd != 0) { + taosCloseSocket(fd); + } + + return code; +} diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 313b879abe5d2980be94dfd3ed348c507777d834..9672d1176d269c09d0d5db38293e28aa00f9db34 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -18,36 +18,73 @@ #include "cJSON.h" #include "taoserror.h" -SJson* tjsonCreateObject() { return cJSON_CreateObject(); } +SJson* tjsonCreateObject() { + SJson* pJson = cJSON_CreateObject(); + if (pJson == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return pJson; +} void tjsonDelete(SJson* pJson) { cJSON_Delete((cJSON*)pJson); } int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number) { char tmp[40] = {0}; - snprintf(tmp, tListLen(tmp), "%" PRId64, number); + snprintf(tmp, sizeof(tmp), "%" PRId64, number); return tjsonAddStringToObject(pJson, pName, tmp); } int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) { - return (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); + if (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; } int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean) { - return (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); + if (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; } int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) { - return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); + if (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; } -SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) { return cJSON_AddArrayToObject((cJSON*)pJson, pName); } +SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) { + SJson* ret = (SJson*)cJSON_AddArrayToObject((cJSON*)pJson, pName); + if (ret == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return ret; +} int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem) { - return (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); + if (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem)) { + return TSDB_CODE_SUCCESS; + } + + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; } int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) { - return (cJSON_AddItemToArray((cJSON*)pJson, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); + if (cJSON_AddItemToArray((cJSON*)pJson, pItem)) { + return TSDB_CODE_SUCCESS; + } + + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; } int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj) { diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 8a1763c4fc8d2f778bf6c0b2874ff16c5868cf48..f62c43773d286a03aa9ad073d21128a5eed3f18d 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -41,9 +41,11 @@ void *cancelHandler(void *arg) { taosReleaseRef(tscObjRef, rid); #endif #else + reset_terminal_mode(); printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); #endif + reset_terminal_mode(); printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); }