From 7baf8eab29e0a00012610d0a97158cac25c8c783 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 20 May 2022 09:53:30 +0800 Subject: [PATCH] catalog async api --- include/libs/catalog/catalog.h | 66 +- source/libs/catalog/inc/catalogInt.h | 176 +- source/libs/catalog/inc/ctgRemote.h | 35 + source/libs/catalog/src/catalog.c | 2731 ++--------------- source/libs/catalog/src/ctgAsync.c | 1015 ++++++ source/libs/catalog/src/ctgCache.c | 1513 +++++++++ .../catalog/src/{catalogDbg.c => ctgDbg.c} | 0 source/libs/catalog/src/ctgRemote.c | 582 ++++ source/libs/catalog/src/ctgUtil.c | 577 ++++ source/libs/catalog/test/catalogTests.cpp | 14 +- source/libs/function/src/functionMgt.c | 20 +- 11 files changed, 4206 insertions(+), 2523 deletions(-) create mode 100644 source/libs/catalog/inc/ctgRemote.h create mode 100644 source/libs/catalog/src/ctgAsync.c create mode 100644 source/libs/catalog/src/ctgCache.c rename source/libs/catalog/src/{catalogDbg.c => ctgDbg.c} (100%) create mode 100644 source/libs/catalog/src/ctgRemote.c create mode 100644 source/libs/catalog/src/ctgUtil.c diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 7b91e9546c..e64fb4235c 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -46,24 +46,34 @@ typedef enum { AUTH_TYPE_OTHER, } AUTH_TYPE; +typedef struct SUserAuthInfo { + char user[TSDB_USER_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + AUTH_TYPE type; +} SUserAuthInfo; + typedef struct SCatalogReq { - SArray *pTableName; // element is SNAME - SArray *pUdf; // udf name + SArray *pTableMeta; // element is SNAME + SArray *pDbVgroup; // element is db full name + SArray *pTableHash; // element is SNAME + SArray *pUdf; // element is udf name + SArray *pDbCfg; // element is db full name + SArray *pIndex; // element is index name + SArray *pUser; // element is SUserAuthInfo bool qNodeRequired; // valid qnode } SCatalogReq; typedef struct SMetaData { - SArray *pTableMeta; // STableMeta array - SArray *pVgroupInfo; // SVgroupInfo list - SArray *pUdfList; // udf info list - SArray *pQnodeList; // qnode list, SArray + SArray *pTableMeta; // SArray + SArray *pDbVgroup; // SArray*> + SArray *pTableHash; // SArray + SArray *pUdfList; // SArray + SArray *pDbCfg; // SArray + SArray *pIndex; // SArray + SArray *pUser; // SArray + SArray *pQnodeList; // SArray } SMetaData; -typedef struct STbSVersion { - char* tbFName; - int32_t sver; -} STbSVersion; - typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; @@ -88,6 +98,11 @@ typedef struct SDbVgVersion { int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SDbVgVersion; +typedef struct STbSVersion { + char* tbFName; + int32_t sver; +} STbSVersion; + typedef struct SUserAuthVersion { char user[TSDB_USER_LEN]; int32_t version; @@ -96,6 +111,8 @@ typedef struct SUserAuthVersion { typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; +typedef void (*catalogCallback)(SMetaData* pResult, void* param, int32_t code); + int32_t catalogInit(SCatalogCfg *cfg); /** @@ -127,32 +144,11 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers */ int32_t catalogGetDBVgInfo(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, SArray** pVgroupList); -typedef struct { -} SResultMetaInfoWrapper; - -typedef void (*__async_cb_fn_t)(const SResultMetaInfoWrapper* pResult, void* param, int32_t code); - -typedef struct { - SCatalog* pCatalog; - void* pTransporter; - SEpSet* pMgmtEps; - char* pDbname; -} CatalogParamWrapper; - -/** - * - * @param pCatalogWrapper - * @param fp - * @param param - * @return - */ -int32_t catalogGetDBVgInfo_a(CatalogParamWrapper* pCatalogWrapper, __async_cb_fn_t fp, void* param); - int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgInfo* dbInfo); int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); -int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName); +int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName); int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid); @@ -262,9 +258,9 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); -int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); +int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); -int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); +int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo); int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 857c708852..1ea53730ac 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -58,6 +58,17 @@ enum { CTG_ACT_MAX }; +typedef enum { + CTG_TASK_GET_QNODE = 0, + CTG_TASK_GET_DB_VGROUP, + CTG_TASK_GET_DB_CFG, + CTG_TASK_GET_TB_META, + CTG_TASK_GET_TB_HASH, + CTG_TASK_GET_INDEX, + CTG_TASK_GET_UDF, + CTG_TASK_GET_USER, +} CTG_TASK_TYPE; + typedef struct SCtgDebug { bool lockEnable; bool cacheEnable; @@ -66,6 +77,43 @@ typedef struct SCtgDebug { uint32_t showCachePeriodSec; } SCtgDebug; +typedef struct SCtgTbCacheInfo { + bool inCache; + uint64_t dbId; + uint64_t suid; + int32_t tbType; +} SCtgTbCacheInfo; + +typedef struct SCtgTbMetaCtx { + SCtgTbCacheInfo tbInfo; + SName* pName; + int32_t flag; +} SCtgTbMetaCtx; + +typedef struct SCtgDbVgCtx { + char dbFName[TSDB_DB_FNAME_LEN]; +} SCtgDbVgCtx; + +typedef struct SCtgDbCfgCtx { + char dbFName[TSDB_DB_FNAME_LEN]; +} SCtgDbCfgCtx; + +typedef struct SCtgTbHashCtx { + char dbFName[TSDB_DB_FNAME_LEN]; + SName* pName; +} SCtgTbHashCtx; + +typedef struct SCtgIndexCtx { + char indexFName[TSDB_INDEX_FNAME_LEN]; +} SCtgIndexCtx; + +typedef struct SCtgUdfCtx { + char udfName[TSDB_FUNC_NAME_LEN]; +} SCtgUdfCtx; + +typedef struct SCtgUserCtx { + SUserAuthInfo user; +} SCtgUserCtx; typedef struct SCtgTbMetaCache { SRWLatch stbLock; @@ -113,6 +161,55 @@ typedef struct SCatalog { SCtgRentMgmt stbRent; } SCatalog; +typedef struct SCtgJob { + int64_t refId; + SArray* pTasks; + int32_t taskDone; + SMetaData jobRes; + int32_t rspCode; + + uint64_t queryId; + SCatalog* pCtg; + void* pTrans; + const SEpSet* pMgmtEps; + void* userParam; + catalogCallback userFp; + int32_t tbMetaNum; + int32_t tbHashNum; + int32_t dbVgNum; + int32_t udfNum; + int32_t qnodeNum; + int32_t dbCfgNum; + int32_t indexNum; + int32_t userNum; +} SCtgJob; + +typedef struct SCtgMsgCtx { + int32_t reqType; + void* lastOut; + void* out; + char* target; +} SCtgMsgCtx; + +typedef struct SCtgTask { + CTG_TASK_TYPE type; + int32_t taskId; + SCtgJob *pJob; + void* taskCtx; + SCtgMsgCtx msgCtx; + void* res; +} SCtgTask; + +typedef int32_t (*ctgLanchTaskFp)(SCtgTask*); +typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTask*, int32_t, const SDataBuf *, int32_t); +typedef int32_t (*ctgDumpTaskResFp)(SCtgTask*); + +typedef struct SCtgAsyncFps { + ctgLanchTaskFp launchFp; + ctgHandleTaskMsgRspFp handleRspFp; + ctgDumpTaskResFp dumpResFp; +} SCtgAsyncFps; + typedef struct SCtgApiStat { #ifdef WINDOWS @@ -214,6 +311,7 @@ typedef struct SCtgQueue { typedef struct SCatalogMgmt { bool exit; + int32_t jobPool; SRWLatch lock; SCtgQueue queue; TdThread updateThread; @@ -327,10 +425,80 @@ typedef struct SCtgAction { #define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0) #define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) - -extern void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p); -extern void ctgdShowClusterCache(SCatalog* pCtg); -extern int32_t ctgdShowCacheInfo(void); +#define CTG_PARAMS SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps +#define CTG_PARAMS_LIST() pCtg, pTrans, pMgmtEps + +void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p); +void ctgdShowClusterCache(SCatalog* pCtg); +int32_t ctgdShowCacheInfo(void); + +int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); +int32_t ctgGetTbMetaFromCache(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); + +int32_t ctgActUpdateVg(SCtgMetaAction *action); +int32_t ctgActUpdateTb(SCtgMetaAction *action); +int32_t ctgActRemoveDB(SCtgMetaAction *action); +int32_t ctgActRemoveStb(SCtgMetaAction *action); +int32_t ctgActRemoveTb(SCtgMetaAction *action); +int32_t ctgActUpdateUser(SCtgMetaAction *action); +int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); +void ctgReleaseVgInfo(SCtgDBCache *dbCache); +int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); +int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist); +int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); +int32_t ctgReadTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver); +int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); +int32_t ctgPutRmDBToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId); +int32_t ctgPutRmStbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq); +int32_t ctgPutRmTbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq); +int32_t ctgPutUpdateVgToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq); +int32_t ctgPutUpdateTbToQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); +int32_t ctgPutUpdateUserToQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); +int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); +int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); +int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); +int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); +int32_t ctgStartUpdateThread(); +int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask); + + + +int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target); +int32_t ctgGetDBVgInfoFromMnode(CTG_PARAMS, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask); +int32_t ctgGetQnodeListFromMnode(CTG_PARAMS, SArray *out, SCtgTask* pTask); +int32_t ctgGetDBCfgFromMnode(CTG_PARAMS, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); +int32_t ctgGetIndexInfoFromMnode(CTG_PARAMS, const char *indexName, SIndexInfo *out, SCtgTask* pTask); +int32_t ctgGetUdfInfoFromMnode(CTG_PARAMS, const char *funcName, SFuncInfo *out, SCtgTask* pTask); +int32_t ctgGetUserDbAuthFromMnode(CTG_PARAMS, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask); +int32_t ctgGetTbMetaFromMnodeImpl(CTG_PARAMS, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); +int32_t ctgGetTbMetaFromMnode(CTG_PARAMS, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask); +int32_t ctgGetTbMetaFromVnode(CTG_PARAMS, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask); + +int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param); +int32_t ctgLaunchJob(SCtgJob *pJob); +int32_t ctgMakeAsyncRes(SCtgJob *pJob); + +int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst); +int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput); +int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList); +void ctgFreeJob(void* job); +void ctgFreeHandle(SCatalog* pCtg); +void ctgFreeVgInfo(SDBVgInfo *vgInfo); +int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup); +void ctgResetTbMetaTask(SCtgTask* pTask); +void ctgFreeDbCache(SCtgDBCache *dbCache); +int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); +int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2); +int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); +int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2); +void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); +int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); + + +extern SCatalogMgmt gCtgMgmt; +extern SCtgDebug gCTGDebug; +extern SCtgAsyncFps gCtgAsyncFps[]; #ifdef __cplusplus } diff --git a/source/libs/catalog/inc/ctgRemote.h b/source/libs/catalog/inc/ctgRemote.h new file mode 100644 index 0000000000..cd88863c1b --- /dev/null +++ b/source/libs/catalog/inc/ctgRemote.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_CATALOG_REMOTE_H_ +#define _TD_CATALOG_REMOTE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SCtgTaskCallbackParam { + uint64_t queryId; + int64_t refId; + uint64_t taskId; + int32_t reqType; +} SCtgTaskCallbackParam; + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_CATALOG_REMOTE_H_*/ diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 0d677cddab..32d7cc9f8e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -18,1792 +18,47 @@ #include "tname.h" #include "catalogInt.h" #include "systable.h" +#include "tref.h" -int32_t ctgActUpdateVg(SCtgMetaAction *action); -int32_t ctgActUpdateTbl(SCtgMetaAction *action); -int32_t ctgActRemoveDB(SCtgMetaAction *action); -int32_t ctgActRemoveStb(SCtgMetaAction *action); -int32_t ctgActRemoveTbl(SCtgMetaAction *action); -int32_t ctgActUpdateUser(SCtgMetaAction *action); - -extern SCtgDebug gCTGDebug; SCatalogMgmt gCtgMgmt = {0}; -SCtgAction gCtgAction[CTG_ACT_MAX] = {{ - CTG_ACT_UPDATE_VG, - "update vgInfo", - ctgActUpdateVg - }, - { - CTG_ACT_UPDATE_TBL, - "update tbMeta", - ctgActUpdateTbl - }, - { - CTG_ACT_REMOVE_DB, - "remove DB", - ctgActRemoveDB - }, - { - CTG_ACT_REMOVE_STB, - "remove stbMeta", - ctgActRemoveStb - }, - { - CTG_ACT_REMOVE_TBL, - "remove tbMeta", - ctgActRemoveTbl - }, - { - CTG_ACT_UPDATE_USER, - "update user", - ctgActUpdateUser - } -}; - -void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { - if (NULL == mgmt->slots) { - return; - } - - for (int32_t i = 0; i < mgmt->slotNum; ++i) { - SCtgRentSlot *slot = &mgmt->slots[i]; - if (slot->meta) { - taosArrayDestroy(slot->meta); - slot->meta = NULL; - } - } - - taosMemoryFreeClear(mgmt->slots); -} - - -void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - int32_t stblNum = taosHashGetSize(cache->stbCache); - taosHashCleanup(cache->stbCache); - cache->stbCache = NULL; - CTG_CACHE_STAT_SUB(stblNum, stblNum); - } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); - - CTG_LOCK(CTG_WRITE, &cache->metaLock); - if (cache->metaCache) { - int32_t tblNum = taosHashGetSize(cache->metaCache); - taosHashCleanup(cache->metaCache); - cache->metaCache = NULL; - CTG_CACHE_STAT_SUB(tblNum, tblNum); - } - CTG_UNLOCK(CTG_WRITE, &cache->metaLock); -} - -void ctgFreeVgInfo(SDBVgInfo *vgInfo) { - if (NULL == vgInfo) { - return; - } - - if (vgInfo->vgHash) { - taosHashCleanup(vgInfo->vgHash); - vgInfo->vgHash = NULL; - } - - taosMemoryFreeClear(vgInfo); -} - -void ctgFreeDbCache(SCtgDBCache *dbCache) { - if (NULL == dbCache) { - return; - } - - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - ctgFreeVgInfo (dbCache->vgInfo); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - - ctgFreeTableMetaCache(&dbCache->tbCache); -} - -void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) { - taosHashCleanup(userCache->createdDbs); - taosHashCleanup(userCache->readDbs); - taosHashCleanup(userCache->writeDbs); -} - -void ctgFreeHandle(SCatalog* pCtg) { - ctgFreeMetaRent(&pCtg->dbRent); - ctgFreeMetaRent(&pCtg->stbRent); - - if (pCtg->dbCache) { - int32_t dbNum = taosHashGetSize(pCtg->dbCache); - - void *pIter = taosHashIterate(pCtg->dbCache, NULL); - while (pIter) { - SCtgDBCache *dbCache = pIter; - - atomic_store_8(&dbCache->deleted, 1); - - ctgFreeDbCache(dbCache); - - pIter = taosHashIterate(pCtg->dbCache, pIter); - } - - taosHashCleanup(pCtg->dbCache); - - CTG_CACHE_STAT_SUB(dbNum, dbNum); - } - - if (pCtg->userCache) { - int32_t userNum = taosHashGetSize(pCtg->userCache); - - void *pIter = taosHashIterate(pCtg->userCache, NULL); - while (pIter) { - SCtgUserAuth *userCache = pIter; - - ctgFreeSCtgUserAuth(userCache); - - pIter = taosHashIterate(pCtg->userCache, pIter); - } - - taosHashCleanup(pCtg->userCache); - - CTG_CACHE_STAT_SUB(userNum, userNum); - } - - taosMemoryFree(pCtg); -} - - - -void ctgWaitAction(SCtgMetaAction *action) { - while (true) { - tsem_wait(&gCtgMgmt.queue.rspSem); - - if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { - tsem_post(&gCtgMgmt.queue.rspSem); - break; - } - - if (gCtgMgmt.queue.seqDone >= action->seqId) { - break; - } - - tsem_post(&gCtgMgmt.queue.rspSem); - sched_yield(); - } -} - -void ctgPopAction(SCtgMetaAction **action) { - SCtgQNode *orig = gCtgMgmt.queue.head; - - SCtgQNode *node = gCtgMgmt.queue.head->next; - gCtgMgmt.queue.head = gCtgMgmt.queue.head->next; - - CTG_QUEUE_SUB(); - - taosMemoryFreeClear(orig); - - *action = &node->action; -} - - -int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { - SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); - if (NULL == node) { - qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); - CTG_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - action->seqId = atomic_add_fetch_64(&gCtgMgmt.queue.seqId, 1); - - node->action = *action; - - CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); - gCtgMgmt.queue.tail->next = node; - gCtgMgmt.queue.tail = node; - CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); - - CTG_QUEUE_ADD(); - CTG_RUNTIME_STAT_ADD(qNum, 1); - - tsem_post(&gCtgMgmt.queue.reqSem); - - ctgDebug("action [%s] added into queue", gCtgAction[action->act].name); - - if (action->syncReq) { - ctgWaitAction(action); - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; - SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - char *p = strchr(dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { - dbFName = p + 1; - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - msg->dbId = dbId; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - taosMemoryFreeClear(action.data); - CTG_RET(code); -} - - -int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq}; - SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - strncpy(msg->stbName, stbName, sizeof(msg->stbName)); - msg->dbId = dbId; - msg->suid = suid; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - taosMemoryFreeClear(action.data); - CTG_RET(code); -} - - - -int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq}; - SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - strncpy(msg->tbName, tbName, sizeof(msg->tbName)); - msg->dbId = dbId; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - taosMemoryFreeClear(action.data); - CTG_RET(code); -} - -int32_t ctgPushUpdateVgMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; - SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); - ctgFreeVgInfo(dbInfo); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - char *p = strchr(dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { - dbFName = p + 1; - } - - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - msg->pCtg = pCtg; - msg->dbId = dbId; - msg->dbInfo = dbInfo; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - ctgFreeVgInfo(dbInfo); - taosMemoryFreeClear(action.data); - CTG_RET(code); -} - -int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq}; - SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - char *p = strchr(output->dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { - memmove(output->dbFName, p + 1, strlen(p + 1)); - } - - msg->pCtg = pCtg; - msg->output = output; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; - SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - msg->userAuth = *pAuth; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(pCtg, &action)); - - return TSDB_CODE_SUCCESS; - -_return: - - tFreeSGetUserAuthRsp(pAuth); - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { - CTG_LOCK(CTG_READ, &dbCache->vgLock); - - if (dbCache->deleted) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - - ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); - - *inCache = false; - return TSDB_CODE_SUCCESS; - } - - - if (NULL == dbCache->vgInfo) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - - *inCache = false; - ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); - return TSDB_CODE_SUCCESS; - } - - *inCache = true; - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - - if (dbCache->deleted) { - ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - - return TSDB_CODE_SUCCESS; -} - -void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { - taosHashRelease(pCtg->dbCache, dbCache); -} - -void ctgReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); -} - -void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); -} - - -int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { - char *p = strchr(dbFName, '.'); - if (p && CTG_IS_SYS_DBNAME(p + 1)) { - dbFName = p + 1; - } - - SCtgDBCache *dbCache = NULL; - if (acquire) { - dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); - } else { - dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); - } - - if (NULL == dbCache) { - *pCache = NULL; - ctgDebug("db not in cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; - } - - if (dbCache->deleted) { - if (acquire) { - ctgReleaseDBCache(pCtg, dbCache); - } - - *pCache = NULL; - ctgDebug("db is removing from cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; - } - - *pCache = dbCache; - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { - CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); -} - -int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { - CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); -} - - -int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { - SCtgDBCache *dbCache = NULL; - - if (NULL == pCtg->dbCache) { - ctgDebug("empty db cache, dbFName:%s", dbFName); - goto _return; - } - - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %s not in cache", dbFName); - goto _return; - } - - ctgAcquireVgInfo(pCtg, dbCache, inCache); - if (!(*inCache)) { - ctgDebug("vgInfo of db %s not in cache", dbFName); - goto _return; - } - - *pCache = dbCache; - *inCache = true; - - CTG_CACHE_STAT_ADD(vgHitNum, 1); - - ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); - - return TSDB_CODE_SUCCESS; - -_return: - - if (dbCache) { - ctgReleaseDBCache(pCtg, dbCache); - } - - *pCache = NULL; - *inCache = false; - - CTG_CACHE_STAT_ADD(vgMissNum, 1); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray *out) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pMgmtEps->inUse); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_QNODE_LIST)](NULL, &msg, 0, &msgLen); - if (code) { - ctgError("Build qnode list msg failed, error:%s", tstrerror(code)); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_QNODE_LIST, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for qnode list, error:%s", tstrerror(rpcRsp.code)); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_QNODE_LIST)](out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process qnode list rsp failed, error:%s", tstrerror(rpcRsp.code)); - CTG_ERR_RET(code); - } - - ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out)); - - return TSDB_CODE_SUCCESS; -} - -//typedef void __taos_async_internal_fn_t(void* param, ) -void ctgGetDBVgInfoFromMnode_a(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, __taos_async_fn_t fp) { - -} - -int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)](input, &msg, 0, &msgLen); - if (code) { - ctgError("Build use db msg failed, code:%x, db:%s", code, input->db); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_USE_DB, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for use db, error:%s, db:%s", tstrerror(rpcRsp.code), input->db); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)](out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process use db rsp failed, code:%x, db:%s", code, input->db); - CTG_ERR_RET(code); - } - - ctgDebug("Got db vgInfo from mnode, dbFName:%s", input->db); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *dbFName, SDbCfgInfo *out) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)]((void *)dbFName, &msg, 0, &msgLen); - if (code) { - ctgError("Build get db cfg msg failed, code:%x, db:%s", code, dbFName); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_GET_DB_CFG, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for get db cfg, error:%s, db:%s", tstrerror(rpcRsp.code), dbFName); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)](out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process get db cfg rsp failed, code:%x, db:%s", code, dbFName); - CTG_ERR_RET(code); - } - - ctgDebug("Got db cfg from mnode, dbFName:%s", dbFName); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *indexName, SIndexInfo *out) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get index from mnode, indexName:%s", indexName); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)]((void *)indexName, &msg, 0, &msgLen); - if (code) { - ctgError("Build get index msg failed, code:%x, db:%s", code, indexName); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_GET_INDEX, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for get index, error:%s, indexName:%s", tstrerror(rpcRsp.code), indexName); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)](out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process get index rsp failed, code:%x, indexName:%s", code, indexName); - CTG_ERR_RET(code); - } - - ctgDebug("Got index from mnode, indexName:%s", indexName); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *funcName, SFuncInfo **out) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get udf info from mnode, funcName:%s", funcName); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)]((void *)funcName, &msg, 0, &msgLen); - if (code) { - ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_RETRIEVE_FUNC, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - if (TSDB_CODE_MND_FUNC_NOT_EXIST == rpcRsp.code) { - ctgDebug("funcName %s not exist in mnode", funcName); - taosMemoryFreeClear(*out); - CTG_RET(TSDB_CODE_SUCCESS); - } - - ctgError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rpcRsp.code), funcName); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)](*out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process get udf rsp failed, code:%x, funcName:%s", code, funcName); - CTG_ERR_RET(code); - } - - ctgDebug("Got udf from mnode, funcName:%s", funcName); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *user, SGetUserAuthRsp *authRsp) { - char *msg = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get user auth from mnode, user:%s", user); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)]((void *)user, &msg, 0, &msgLen); - if (code) { - ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_GET_USER_AUTH, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for get user auth, error:%s, user:%s", tstrerror(rpcRsp.code), user); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)](authRsp, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process get user auth rsp failed, code:%x, user:%s", code, user); - CTG_ERR_RET(code); - } - - ctgDebug("Got user auth from mnode, user:%s", user); - - return TSDB_CODE_SUCCESS; -} - - -int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { - if (NULL == pCtg->dbCache) { - *exist = 0; - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; - } - - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - *exist = 0; - return TSDB_CODE_SUCCESS; - } - - size_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - - *exist = 0; - ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; - } - - *exist = 1; - - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); - - return TSDB_CODE_SUCCESS; -} - - -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, bool *inCache, int32_t flag, uint64_t *dbId) { - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); - goto _return; - } - - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - if (CTG_FLAG_IS_SYS_DB(flag)) { - strcpy(dbFName, pTableName->dbname); - } else { - tNameGetFullDbName(pTableName, dbFName); - } - - *pTableMeta = NULL; - - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %s not in cache", pTableName->tname); - goto _return; - } - - int32_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), (void **)pTableMeta, &sz); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == *pTableMeta) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); - goto _return; - } - - if (dbId) { - *dbId = dbCache->dbId; - } - - STableMeta* tbMeta = *pTableMeta; - - if (tbMeta->tableType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); - - *inCache = true; - CTG_CACHE_STAT_ADD(tblHitNum, 1); - - return TSDB_CODE_SUCCESS; - } - - ctgDebug("Got subtable meta from cache, type:%d, dbFName:%s, tbName:%s, suid:%" PRIx64, tbMeta->tableType, dbFName, pTableName->tname, tbMeta->suid); - - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); - if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); - taosMemoryFreeClear(*pTableMeta); - goto _return; - } - - if ((*stbMeta)->suid != tbMeta->suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - taosMemoryFreeClear(*pTableMeta); - ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - int32_t metaSize = CTG_META_SIZE(*stbMeta); - *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); - if (NULL == *pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgError("realloc size[%d] failed", metaSize); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); - - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - - ctgReleaseDBCache(pCtg, dbCache); - - *inCache = true; - CTG_CACHE_STAT_ADD(tblHitNum, 1); - - ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); - - return TSDB_CODE_SUCCESS; - -_return: - - *inCache = false; - CTG_CACHE_STAT_ADD(tblMissNum, 1); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { - if (NULL == pCtg->dbCache) { - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); - return TSDB_CODE_SUCCESS; - } - - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName)); - - if (NULL == pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); - ctgReleaseDBCache(pCtg, dbCache); - - return TSDB_CODE_SUCCESS; - } - - *tbType = atomic_load_8(&pTableMeta->tableType); - - taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); - - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { - if (NULL == pCtg->userCache) { - ctgDebug("empty user auth cache, user:%s", user); - goto _return; - } - - SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); - if (NULL == pUser) { - ctgDebug("user not in cache, user:%s", user); - goto _return; - } - - *inCache = true; - - ctgDebug("Got user from cache, user:%s", user); - CTG_CACHE_STAT_ADD(userHitNum, 1); - - if (pUser->superUser) { - *pass = true; - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_READ, &pUser->lock); - if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { - *pass = true; - CTG_UNLOCK(CTG_READ, &pUser->lock); - return TSDB_CODE_SUCCESS; - } - - if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { - *pass = true; - } - - if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { - *pass = true; - } - - CTG_UNLOCK(CTG_READ, &pUser->lock); - - return TSDB_CODE_SUCCESS; - -_return: - - *inCache = false; - CTG_CACHE_STAT_ADD(userMissNum, 1); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { - SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; - char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; - int32_t msgLen = 0; - - ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)](&bInput, &msg, 0, &msgLen); - if (code) { - ctgError("Build mnode stablemeta msg failed, code:%x", code); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TABLE_META, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { - SET_META_TYPE_NULL(output->metaType); - ctgDebug("stablemeta not exist in mnode, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; - } - - ctgError("error rsp for stablemeta from mnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tbName); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process mnode stablemeta rsp failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tbName); - CTG_ERR_RET(code); - } - - ctgDebug("Got table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); - - return ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, dbFName, (char *)pTableName->tname, output); -} - -int32_t ctgGetTableMetaFromVnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); - - ctgDebug("try to get table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; - char *msg = NULL; - int32_t msgLen = 0; - - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen); - if (code) { - ctgError("Build vnode tablemeta msg failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tNameGetTableName(pTableName)); - CTG_ERR_RET(code); - } - - SRpcMsg rpcMsg = { - .msgType = TDMT_VND_TABLE_META, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); - - if (TSDB_CODE_SUCCESS != rpcRsp.code) { - if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { - SET_META_TYPE_NULL(output->metaType); - ctgDebug("tablemeta not exist in vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - return TSDB_CODE_SUCCESS; - } - - ctgError("error rsp for table meta from vnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tNameGetTableName(pTableName)); - CTG_ERR_RET(rpcRsp.code); - } - - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - ctgError("Process vnode tablemeta rsp failed, code:%s, dbFName:%s, tbName:%s", tstrerror(code), dbFName, tNameGetTableName(pTableName)); - CTG_ERR_RET(code); - } - - ctgDebug("Got table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - int32_t code = 0; - int32_t retryNum = 0; - - while (retryNum < CTG_DEFAULT_MAX_RETRY_TIMES) { - code = ctgGetTableMetaFromVnodeImpl(pCtg, pTrans, pMgmtEps, pTableName, vgroupInfo, output); - if (code) { - if (TSDB_CODE_VND_HASH_MISMATCH == code) { - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); - - code = catalogRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName); - if (code != TSDB_CODE_SUCCESS) { - break; - } - - ++retryNum; - continue; - } - } - - break; - } - - CTG_RET(code); -} - -int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { - switch (hashMethod) { - default: - *fp = MurmurHash3_32; - break; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) { - SHashObj *vgroupHash = NULL; - SVgroupInfo *vgInfo = NULL; - SArray *vgList = NULL; - int32_t code = 0; - int32_t vgNum = taosHashGetSize(vgHash); - - vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); - if (NULL == vgList) { - ctgError("taosArrayInit failed, num:%d", vgNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - void *pIter = taosHashIterate(vgHash, NULL); - while (pIter) { - vgInfo = pIter; - - if (NULL == taosArrayPush(vgList, vgInfo)) { - ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); - taosHashCancelIterate(vgHash, pIter); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - pIter = taosHashIterate(vgHash, pIter); - vgInfo = NULL; - } - - *pList = vgList; - - ctgDebug("Got vgList from cache, vgNum:%d", vgNum); - - return TSDB_CODE_SUCCESS; - -_return: - - if (vgList) { - taosArrayDestroy(vgList); - } - - CTG_RET(code); -} - -int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { - int32_t code = 0; - - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); - char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); - - if (vgNum <= 0) { - ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum); - CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); - } - - tableNameHashFp fp = NULL; - SVgroupInfo *vgInfo = NULL; - - CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); - - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); - - uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); - - void *pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - vgInfo = pIter; - if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { - taosHashCancelIterate(dbInfo->vgHash, pIter); - break; - } - - pIter = taosHashIterate(dbInfo->vgHash, pIter); - vgInfo = NULL; - } - - if (NULL == vgInfo) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - *pVgroup = *vgInfo; - - CTG_RET(code); -} - -int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { - if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { - return -1; - } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { - return 1; - } else { - return 0; - } -} - -int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { - if (*(int64_t *)key1 < ((SDbVgVersion*)key2)->dbId) { - return -1; - } else if (*(int64_t *)key1 > ((SDbVgVersion*)key2)->dbId) { - return 1; - } else { - return 0; - } -} - -int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { - return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { - return 1; - } else { - return 0; - } -} - -int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) { - if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { - return -1; - } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { - return 1; - } else { - return 0; - } -} - - -int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { - mgmt->slotRIdx = 0; - mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; - mgmt->type = type; - - size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; - - mgmt->slots = taosMemoryCalloc(1, msgSize); - if (NULL == mgmt->slots) { - qError("calloc %d failed", (int32_t)msgSize); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); - - return TSDB_CODE_SUCCESS; -} - - -int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { - int16_t widx = abs((int)(id % mgmt->slotNum)); - - SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - - CTG_LOCK(CTG_WRITE, &slot->lock); - if (NULL == slot->meta) { - slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size); - if (NULL == slot->meta) { - qError("taosArrayInit %d failed, id:%"PRIx64", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - } - - if (NULL == taosArrayPush(slot->meta, meta)) { - qError("taosArrayPush meta to rent failed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - slot->needSort = true; - - qDebug("add meta to rent, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); - -_return: - - CTG_UNLOCK(CTG_WRITE, &slot->lock); - CTG_RET(code); -} - -int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { - int16_t widx = abs((int)(id % mgmt->slotNum)); - - SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - - CTG_LOCK(CTG_WRITE, &slot->lock); - if (NULL == slot->meta) { - qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - if (slot->needSort) { - qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); - taosArraySort(slot->meta, sortCompare); - slot->needSort = false; - qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); - } - - void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); - if (NULL == orig) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - memcpy(orig, meta, size); - - qDebug("meta in rent updated, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); - -_return: - - CTG_UNLOCK(CTG_WRITE, &slot->lock); - - if (code) { - qWarn("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); - CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); - } - - CTG_RET(code); -} - -int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { - int16_t widx = abs((int)(id % mgmt->slotNum)); - - SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - - CTG_LOCK(CTG_WRITE, &slot->lock); - if (NULL == slot->meta) { - qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - if (slot->needSort) { - taosArraySort(slot->meta, sortCompare); - slot->needSort = false; - qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type); - } - - int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ); - if (idx < 0) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - taosArrayRemove(slot->meta, idx); - - qDebug("meta in rent removed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); - -_return: - - CTG_UNLOCK(CTG_WRITE, &slot->lock); - - CTG_RET(code); -} - - -int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { - int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1); - if (ridx >= mgmt->slotNum) { - ridx %= mgmt->slotNum; - atomic_store_16(&mgmt->slotRIdx, ridx); - } - - SCtgRentSlot *slot = &mgmt->slots[ridx]; - int32_t code = 0; - - CTG_LOCK(CTG_READ, &slot->lock); - if (NULL == slot->meta) { - qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type); - *num = 0; - goto _return; - } - - size_t metaNum = taosArrayGetSize(slot->meta); - if (metaNum <= 0) { - qDebug("no meta in slot:%d, type:%d", ridx, mgmt->type); - *num = 0; - goto _return; - } - - size_t msize = metaNum * size; - *res = taosMemoryMalloc(msize); - if (NULL == *res) { - qError("malloc %d failed", (int32_t)msize); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - void *meta = taosArrayGet(slot->meta, 0); - - memcpy(*res, meta, msize); - - *num = (uint32_t)metaNum; - - qDebug("Got %d meta from rent, type:%d", (int32_t)metaNum, mgmt->type); - -_return: - - CTG_UNLOCK(CTG_READ, &slot->lock); - - CTG_RET(code); -} - -int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { - while (true) { - int64_t msec = taosGetTimestampMs(); - int64_t lsec = atomic_load_64(&mgmt->lastReadMsec); - if ((msec - lsec) < CTG_RENT_SLOT_SECOND * 1000) { - *res = NULL; - *num = 0; - qDebug("too short time period to get expired meta, type:%d", mgmt->type); - return TSDB_CODE_SUCCESS; - } - - if (lsec != atomic_val_compare_exchange_64(&mgmt->lastReadMsec, lsec, msec)) { - continue; - } - - break; - } - - CTG_ERR_RET(ctgMetaRentGetImpl(mgmt, res, num, size)); - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { - int32_t code = 0; - - SCtgDBCache newDBCache = {0}; - newDBCache.dbId = dbId; - - newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.metaCache) { - ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.stbCache) { - ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache)); - if (code) { - if (HASH_NODE_EXIST(code)) { - ctgDebug("db already in cache, dbFName:%s", dbFName); - goto _return; - } - - ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); - } - - CTG_CACHE_STAT_ADD(dbNum, 1); - - SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - - ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); - - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); - - return TSDB_CODE_SUCCESS; - -_return: - - ctgFreeDbCache(&newDBCache); - - CTG_RET(code); -} - - -void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - void *pIter = taosHashIterate(cache->stbCache, NULL); - while (pIter) { - uint64_t *suid = NULL; - suid = taosHashGetKey(pIter, NULL); - - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { - ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); - } - - pIter = taosHashIterate(cache->stbCache, pIter); - } - } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); -} - - -int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { - uint64_t dbId = dbCache->dbId; - - ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); - - atomic_store_8(&dbCache->deleted, 1); - - ctgRemoveStbRent(pCtg, &dbCache->tbCache); - - ctgFreeDbCache(dbCache); - - CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); - - if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { - ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - - CTG_CACHE_STAT_SUB(dbNum, 1); - - ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - - return TSDB_CODE_SUCCESS; -} - - -int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { - int32_t code = 0; - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, dbFName, &dbCache); - - if (dbCache) { - // TODO OPEN IT -#if 0 - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#else - if (0 == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbId && (dbCache->dbId == 0)) { - dbCache->dbId = dbId; - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#endif - CTG_ERR_RET(ctgRemoveDB(pCtg, dbCache, dbFName)); - } - - CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); - - ctgGetDBCache(pCtg, dbFName, &dbCache); - - *pCache = dbCache; - - return TSDB_CODE_SUCCESS; -} - - -int32_t ctgUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { - int32_t code = 0; - SDBVgInfo* dbInfo = *pDbInfo; - - if (NULL == dbInfo->vgHash) { - return TSDB_CODE_SUCCESS; - } - - if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", - dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; - - SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - SDBVgInfo *vgInfo = NULL; - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - - if (dbCache->vgInfo) { - if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) { - ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } - - if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) { - ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } - - ctgFreeVgInfo(dbCache->vgInfo); - } - - dbCache->vgInfo = dbInfo; - - *pDbInfo = NULL; - - ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); - - ctgWReleaseVgInfo(dbCache); - - dbCache = NULL; - - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - CTG_RET(code); -} - - -int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { - SCtgTbMetaCache *tbCache = &dbCache->tbCache; - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - - int8_t origType = 0; - uint64_t origSuid = 0; - bool isStb = meta->tableType == TSDB_SUPER_TABLE; - STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (orig) { - origType = orig->tableType; - - if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - return TSDB_CODE_SUCCESS; - } - - if (origType == TSDB_SUPER_TABLE) { - if ((!isStb) || orig->suid != meta->suid) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - } else { - CTG_CACHE_STAT_SUB(stblNum, 1); - } - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); - } - - origSuid = orig->suid; - } - } - - if (isStb) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - } - - if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { - if (isStb) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - } - - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - if (NULL == orig) { - CTG_CACHE_STAT_ADD(tblNum, 1); - } - ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64, dbFName, tbName, meta->tableType, meta->suid); - ctgdShowTableMeta(pCtg, tbName, meta); +int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { + int32_t code = 0; + STableMeta *tblMeta = NULL; + SCtgTbMetaCtx tbCtx = {0}; + tbCtx.flag = CTG_FLAG_UNKNOWN_STB; + tbCtx.pName = pTableName; + + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &tbCtx, &tblMeta)); - if (!isStb) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + if (NULL == tblMeta) { + ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname); return TSDB_CODE_SUCCESS; } - STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - CTG_CACHE_STAT_ADD(stblNum, 1); - - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - - ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64 ",ma:%p", dbFName, tbName, meta->tableType, meta->suid, tbMeta); - - SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); - return TSDB_CODE_SUCCESS; -} - -int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { - *dst = taosMemoryMalloc(sizeof(SDBVgInfo)); - if (NULL == *dst) { - qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - memcpy(*dst, src, sizeof(SDBVgInfo)); - - size_t hashSize = taosHashGetSize(src->vgHash); - (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == (*dst)->vgHash) { - qError("taosHashInit %d failed", (int32_t)hashSize); - taosMemoryFreeClear(*dst); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - int32_t *vgId = NULL; - void *pIter = taosHashIterate(src->vgHash, NULL); - while (pIter) { - vgId = taosHashGetKey(pIter, NULL); - - if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) { - qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); - taosHashCancelIterate(src->vgHash, pIter); - taosHashCleanup((*dst)->vgHash); - taosMemoryFreeClear(*dst); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - pIter = taosHashIterate(src->vgHash, pIter); + if (TSDB_SUPER_TABLE == tblMeta->tableType) { + CTG_ERR_JRET(ctgPutRmStbToQueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, tblMeta->suid, syncReq)); + } else { + CTG_ERR_JRET(ctgPutRmTbToQueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, syncReq)); } + +_return: + taosMemoryFreeClear(tblMeta); - return TSDB_CODE_SUCCESS; + CTG_RET(code); } -void ctgGetDBVgInfo_a(); - -int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { - bool inCache = false; +int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { int32_t code = 0; - CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache)); - if (inCache) { + if (*dbCache) { return TSDB_CODE_SUCCESS; } @@ -1813,19 +68,11 @@ int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - code = ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut); - if (code) { - if (CTG_DB_NOT_EXIST(code) && input.vgVersion > CTG_DEFAULT_INVALID_VERSION) { - ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId); - ctgPushRmDBMsgInQueue(pCtg, input.db, input.dbId); - } - - CTG_ERR_RET(code); - } + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, &DbOut, NULL)); CTG_ERR_JRET(ctgCloneVgInfo(DbOut.dbVgroup, pInfo)); - CTG_ERR_RET(ctgPushUpdateVgMsgInQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, false)); + CTG_ERR_RET(ctgPutUpdateVgToQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, false)); return TSDB_CODE_SUCCESS; @@ -1837,18 +84,17 @@ _return: CTG_RET(code); } -int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName) { - bool inCache = false; +int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { int32_t code = 0; SCtgDBCache* dbCache = NULL; - CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); - if (inCache) { + if (NULL != dbCache) { input.dbId = dbCache->dbId; ctgReleaseVgInfo(dbCache); @@ -1858,59 +104,29 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, c input.vgVersion = CTG_DEFAULT_INVALID_VERSION; input.numOfTable = 0; - code = ctgGetDBVgInfoFromMnode(pCtg, pRpc, pMgmtEps, &input, &DbOut); + code = ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, &DbOut, NULL); if (code) { - if (CTG_DB_NOT_EXIST(code) && inCache) { + if (CTG_DB_NOT_EXIST(code) && (NULL != dbCache)) { ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId); - ctgPushRmDBMsgInQueue(pCtg, input.db, input.dbId); + ctgPutRmDBToQueue(pCtg, input.db, input.dbId); } CTG_ERR_RET(code); } - CTG_ERR_RET(ctgPushUpdateVgMsgInQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, true)); - - return TSDB_CODE_SUCCESS; -} - - - -int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { - *pOutput = taosMemoryMalloc(sizeof(STableMetaOutput)); - if (NULL == *pOutput) { - qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - memcpy(*pOutput, output, sizeof(STableMetaOutput)); - - if (output->tbMeta) { - int32_t metaSize = CTG_META_SIZE(output->tbMeta); - (*pOutput)->tbMeta = taosMemoryMalloc(metaSize); - if (NULL == (*pOutput)->tbMeta) { - qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); - taosMemoryFreeClear(*pOutput); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize); - } + CTG_ERR_RET(ctgPutUpdateVgToQueue(pCtg, dbFName, DbOut.dbId, DbOut.dbVgroup, true)); return TSDB_CODE_SUCCESS; } -int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t flag, STableMetaOutput **pOutput, bool syncReq) { - if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - +int32_t ctgRefreshTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMetaOutput **pOutput, bool syncReq) { SVgroupInfo vgroupInfo = {0}; int32_t code = 0; - if (!CTG_FLAG_IS_SYS_DB(flag)) { - CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); + if (!CTG_FLAG_IS_SYS_DB(ctx->flag)) { + CTG_ERR_RET(catalogGetTableHashVgroup(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo)); } STableMetaOutput moutput = {0}; @@ -1920,39 +136,39 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - if (CTG_FLAG_IS_SYS_DB(flag)) { - ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(pTableName)); + if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { + ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(ctx->pName)); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, (char *)pTableName->tname, output)); - } else if (CTG_FLAG_IS_STB(flag)) { - ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), (char *)ctx->pName->dbname, (char *)ctx->pName->tname, output, NULL)); + } else if (CTG_FLAG_IS_STB(ctx->flag)) { + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); // if get from mnode failed, will not try vnode - CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTrans, pMgmtEps, pTableName, output)); + CTG_ERR_JRET(ctgGetTbMetaFromMnode(CTG_PARAMS_LIST(), ctx->pName, output, NULL)); if (CTG_IS_META_NULL(output->metaType)) { - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo, output, NULL)); } } else { - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pTableName), flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); // if get from vnode failed or no table meta, will not try mnode - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgroupInfo, output, NULL)); if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { - ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pTableName)); + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName)); taosMemoryFreeClear(output->tbMeta); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), output->dbFName, output->tbName, output, NULL)); } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; - if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { - CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); + if (!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) { + CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); } if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput)); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), output->dbFName, output->tbName, &moutput, NULL)); if (CTG_IS_META_NULL(moutput.metaType)) { SET_META_TYPE_NULL(output->metaType); @@ -1970,8 +186,8 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } if (CTG_IS_META_NULL(output->metaType)) { - ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pTableName)); - catalogRemoveTableMeta(pCtg, pTableName); + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName)); + ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1981,471 +197,179 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); } - if (pOutput) { - CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); - } - - CTG_ERR_JRET(ctgPushUpdateTblMsgInQueue(pCtg, output, syncReq)); - - return TSDB_CODE_SUCCESS; - -_return: - - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); - - CTG_RET(code); -} - -int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t flag) { - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - - bool inCache = false; - int32_t code = 0; - uint64_t dbId = 0; - uint64_t suid = 0; - STableMetaOutput *output = NULL; - - if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { - CTG_FLAG_SET_SYS_DB(flag); - } - - CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &inCache, flag, &dbId)); - - int32_t tbType = 0; - - if (inCache) { - if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_SYS_DB(flag)))) { - goto _return; - } - - tbType = (*pTableMeta)->tableType; - suid = (*pTableMeta)->suid; - - taosMemoryFreeClear(*pTableMeta); - } - - if (CTG_FLAG_IS_UNKNOWN_STB(flag)) { - CTG_FLAG_SET_STB(flag, tbType); - } - - - while (true) { - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output, false)); - - if (CTG_IS_META_TABLE(output->metaType)) { - *pTableMeta = output->tbMeta; - goto _return; - } - - if (CTG_IS_META_BOTH(output->metaType)) { - memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); - - *pTableMeta = output->tbMeta; - goto _return; - } - - if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { - ctgError("invalid metaType:%d", output->metaType); - taosMemoryFreeClear(output->tbMeta); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - // HANDLE ONLY CHILD TABLE META - - SName stbName = *pTableName; - strcpy(stbName.tname, output->tbName); - - taosMemoryFreeClear(output->tbMeta); - - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL)); - if (!inCache) { - ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); - - continue; - } - - memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); - - break; - } - -_return: - - if (CTG_TABLE_NOT_EXIST(code) && inCache) { - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - if (CTG_FLAG_IS_SYS_DB(flag)) { - strcpy(dbFName, pTableName->dbname); - } else { - tNameGetFullDbName(pTableName, dbFName); - } - - if (TSDB_SUPER_TABLE == tbType) { - ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid, false); - } else { - ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, false); - } - } - - taosMemoryFreeClear(output); - - if (*pTableMeta) { - ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); - ctgdShowTableMeta(pCtg, pTableName->tname, *pTableMeta); - } - - CTG_RET(code); -} - -int32_t ctgChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { - bool inCache = false; - int32_t code = 0; - - *pass = false; - - CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); - - if (inCache) { - return TSDB_CODE_SUCCESS; - } - - SGetUserAuthRsp authRsp = {0}; - CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp)); - - if (authRsp.superAuth) { - *pass = true; - goto _return; - } - - if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) { - *pass = true; - goto _return; - } - - if (authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { - *pass = true; - } - - if (authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { - *pass = true; - } - -_return: - - ctgPushUpdateUserMsgInQueue(pCtg, &authRsp, false); - - return TSDB_CODE_SUCCESS; -} - - -int32_t ctgActUpdateVg(SCtgMetaAction *action) { - int32_t code = 0; - SCtgUpdateVgMsg *msg = action->data; - - CTG_ERR_JRET(ctgUpdateDBVgInfo(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); - -_return: - - ctgFreeVgInfo(msg->dbInfo); - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - -int32_t ctgActRemoveDB(SCtgMetaAction *action) { - int32_t code = 0; - SCtgRemoveDBMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; - - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); - if (NULL == dbCache) { - goto _return; - } - - if (dbCache->dbId != msg->dbId) { - ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); - goto _return; - } - - CTG_ERR_JRET(ctgRemoveDB(pCtg, dbCache, msg->dbFName)); - -_return: - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - - -int32_t ctgActUpdateTbl(SCtgMetaAction *action) { - int32_t code = 0; - SCtgUpdateTblMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; - STableMetaOutput* output = msg->output; - SCtgDBCache *dbCache = NULL; - - if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { - ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - int32_t metaSize = CTG_META_SIZE(output->tbMeta); - - CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); - } - - if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); - } - -_return: - - if (output) { - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); - } - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - - -int32_t ctgActRemoveStb(SCtgMetaAction *action) { - int32_t code = 0; - SCtgRemoveStbMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; - - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); - if (NULL == dbCache) { - return TSDB_CODE_SUCCESS; - } - - if (msg->dbId && (dbCache->dbId != msg->dbId)) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); - return TSDB_CODE_SUCCESS; - } - - CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { - ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - } else { - CTG_CACHE_STAT_SUB(stblNum, 1); - } - - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { - ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - } else { - CTG_CACHE_STAT_SUB(tblNum, 1); + if (pOutput) { + CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - - ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); - - ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - + CTG_ERR_JRET(ctgPutUpdateTbToQueue(pCtg, output, syncReq)); + + return TSDB_CODE_SUCCESS; + _return: - taosMemoryFreeClear(msg); + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); CTG_RET(code); } -int32_t ctgActRemoveTbl(SCtgMetaAction *action) { - int32_t code = 0; - SCtgRemoveTblMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; - - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); - if (NULL == dbCache) { - return TSDB_CODE_SUCCESS; +int32_t ctgGetTbMetaFromCache(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { + if (CTG_IS_SYS_DBNAME(ctx->pName->dbname)) { + CTG_FLAG_SET_SYS_DB(ctx->flag); } - if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); - return TSDB_CODE_SUCCESS; + CTG_ERR_RET(ctgReadTbMetaFromCache(pCtg, ctx, pTableMeta)); + + if (*pTableMeta) { + if (CTG_FLAG_MATCH_STB(ctx->flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) || (CTG_FLAG_IS_SYS_DB(ctx->flag)))) { + return TSDB_CODE_SUCCESS; + } + + taosMemoryFreeClear(*pTableMeta); } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } else { - CTG_CACHE_STAT_SUB(tblNum, 1); + if (CTG_FLAG_IS_UNKNOWN_STB(ctx->flag)) { + CTG_FLAG_SET_STB(ctx->flag, ctx->tbInfo.tbType); } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + return TSDB_CODE_SUCCESS; +} - ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); -_return: +int32_t ctgGetTbMeta(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { + int32_t code = 0; + STableMetaOutput *output = NULL; - taosMemoryFreeClear(msg); + CTG_ERR_RET(ctgGetTbMetaFromCache(CTG_PARAMS_LIST(), ctx, pTableMeta)); - CTG_RET(code); -} + while (true) { + CTG_ERR_JRET(ctgRefreshTbMeta(CTG_PARAMS_LIST(), ctx, &output, false)); -int32_t ctgActUpdateUser(SCtgMetaAction *action) { - int32_t code = 0; - SCtgUpdateUserMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; - - if (NULL == pCtg->userCache) { - pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - if (NULL == pCtg->userCache) { - ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + if (CTG_IS_META_TABLE(output->metaType)) { + *pTableMeta = output->tbMeta; + goto _return; } - } - - SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); - if (NULL == pUser) { - SCtgUserAuth userAuth = {0}; - - userAuth.version = msg->userAuth.version; - userAuth.superUser = msg->userAuth.superAuth; - userAuth.createdDbs = msg->userAuth.createdDbs; - userAuth.readDbs = msg->userAuth.readDbs; - userAuth.writeDbs = msg->userAuth.writeDbs; - - if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { - ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + + if (CTG_IS_META_BOTH(output->metaType)) { + memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + + *pTableMeta = output->tbMeta; + goto _return; + } + + if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { + ctgError("invalid metaType:%d", output->metaType); + taosMemoryFreeClear(output->tbMeta); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - taosMemoryFreeClear(msg); + // HANDLE ONLY CHILD TABLE META - return TSDB_CODE_SUCCESS; - } + taosMemoryFreeClear(output->tbMeta); - pUser->version = msg->userAuth.version; + SName stbName = *ctx->pName; + strcpy(stbName.tname, output->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = ctx->flag; + stbCtx.pName = &stbName; + + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, pTableMeta)); + if (NULL == *pTableMeta) { + ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, ctx->pName->tname); + continue; + } - CTG_LOCK(CTG_WRITE, &pUser->lock); + memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); - taosHashCleanup(pUser->createdDbs); - pUser->createdDbs = msg->userAuth.createdDbs; - msg->userAuth.createdDbs = NULL; + break; + } - taosHashCleanup(pUser->readDbs); - pUser->readDbs = msg->userAuth.readDbs; - msg->userAuth.readDbs = NULL; +_return: - taosHashCleanup(pUser->writeDbs); - pUser->writeDbs = msg->userAuth.writeDbs; - msg->userAuth.writeDbs = NULL; + if (CTG_TABLE_NOT_EXIST(code) && ctx->tbInfo.inCache) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { + strcpy(dbFName, ctx->pName->dbname); + } else { + tNameGetFullDbName(ctx->pName, dbFName); + } - CTG_UNLOCK(CTG_WRITE, &pUser->lock); + if (TSDB_SUPER_TABLE == ctx->tbInfo.tbType) { + ctgPutRmStbToQueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false); + } else { + ctgPutRmTbToQueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false); + } + } -_return: + taosMemoryFreeClear(output); + if (*pTableMeta) { + ctgDebug("tbmeta returned, tbName:%s, tbType:%d", ctx->pName->tname, (*pTableMeta)->tableType); + ctgdShowTableMeta(pCtg, ctx->pName->tname, *pTableMeta); + } - taosHashCleanup(msg->userAuth.createdDbs); - taosHashCleanup(msg->userAuth.readDbs); - taosHashCleanup(msg->userAuth.writeDbs); - - taosMemoryFreeClear(msg); - CTG_RET(code); } -void* ctgUpdateThreadFunc(void* param) { - setThreadName("catalog"); - - qInfo("catalog update thread started"); - - CTG_LOCK(CTG_READ, &gCtgMgmt.lock); +int32_t ctgChkAuth(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { + bool inCache = false; + int32_t code = 0; - while (true) { - if (tsem_wait(&gCtgMgmt.queue.reqSem)) { - qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); - } - - if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { - tsem_post(&gCtgMgmt.queue.rspSem); - break; - } - - SCtgMetaAction *action = NULL; - ctgPopAction(&action); - SCatalog *pCtg = ((SCtgUpdateMsgHeader *)action->data)->pCtg; - - ctgDebug("process [%s] action", gCtgAction[action->act].name); - - (*gCtgAction[action->act].func)(action); - - gCtgMgmt.queue.seqDone = action->seqId; - - if (action->syncReq) { - tsem_post(&gCtgMgmt.queue.rspSem); - } + *pass = false; + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); - CTG_RUNTIME_STAT_ADD(qDoneNum, 1); + if (inCache) { + return TSDB_CODE_SUCCESS; + } - ctgdShowClusterCache(pCtg); + SGetUserAuthRsp authRsp = {0}; + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(CTG_PARAMS_LIST(), user, &authRsp, NULL)); + + if (authRsp.superAuth) { + *pass = true; + goto _return; } - CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); + if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + goto _return; + } - qInfo("catalog update thread stopped"); - - return NULL; -} + if (type == AUTH_TYPE_READ && authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName))) { + *pass = true; + } else if (type == AUTH_TYPE_WRITE && authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName))) { + *pass = true; + } +_return: -int32_t ctgStartUpdateThread() { - TdThreadAttr thAttr; - taosThreadAttrInit(&thAttr); - taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + ctgPutUpdateUserToQueue(pCtg, &authRsp, false); - if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - CTG_ERR_RET(terrno); - } - - taosThreadAttrDestroy(&thAttr); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { +int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SName* pTableName, SArray** pVgList) { STableMeta *tbMeta = NULL; int32_t code = 0; SVgroupInfo vgroupInfo = {0}; SCtgDBCache* dbCache = NULL; SArray *vgList = NULL; SDBVgInfo *vgInfo = NULL; + SCtgTbMetaCtx ctx = {0}; + ctx.pName = pTableName; + ctx.flag = CTG_FLAG_UNKNOWN_STB; *pVgList = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, &tbMeta, CTG_FLAG_UNKNOWN_STB)); + CTG_ERR_JRET(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, &tbMeta)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); SHashObj *vgHash = NULL; - CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, &dbCache, &vgInfo)); + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, db, &dbCache, &vgInfo)); if (dbCache) { vgHash = dbCache->vgInfo->vgHash; @@ -2561,6 +485,12 @@ int32_t catalogInit(SCatalogCfg *cfg) { } gCtgMgmt.queue.tail = gCtgMgmt.queue.head; + gCtgMgmt.jobPool = taosOpenRef(200, ctgFreeJob); + if (gCtgMgmt.jobPool < 0) { + qError("taosOpenRef failed, error:%s", tstrerror(terrno)); + CTG_ERR_RET(terrno); + } + CTG_ERR_RET(ctgStartUpdateThread()); qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); @@ -2607,6 +537,12 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } + SHashObj *metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == metaCache) { + qError("taosHashInit failed, num:%d", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { if (HASH_NODE_EXIST(code)) { @@ -2663,11 +599,10 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers } SCtgDBCache *dbCache = NULL; - bool inCache = false; int32_t code = 0; - CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); - if (!inCache) { + CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + if (NULL == dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; CTG_API_LEAVE(TSDB_CODE_SUCCESS); } @@ -2688,10 +623,10 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { +int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { + if (NULL == pCtg || NULL == dbFName || NULL == pTrans || NULL == pMgmtEps || NULL == vgroupList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2700,7 +635,7 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, c SArray *vgList = NULL; SHashObj *vgHash = NULL; SDBVgInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName, &dbCache, &vgInfo)); + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName, &dbCache, &vgInfo)); if (dbCache) { vgHash = dbCache->vgInfo->vgHash; } else { @@ -2727,59 +662,6 @@ _return: CTG_API_LEAVE(code); } -typedef struct SCatalogReqObj { - __async_cb_fn_t userFp; - SResultMetaInfoWrapper *pResult; - void* pUserParam; -} SCatalogReqObj; - -void* doExtractpayload(const SDataBuf* pMsg){} - -int32_t loadRemoteMetaCallback(void* param, const SDataBuf* pMsg, int32_t code) { - SCatalogReqObj* pRsp = (SCatalogReqObj*) param; - - // do handle the result. - // serialize data in pMsg and set the result into pRsp->pResult - pRsp->pResult = doExtractpayload(pMsg); - - //todo - - - - // call user's callback function - pRsp->userFp(pRsp->pResult, pRsp->pUserParam, code); - return TSDB_CODE_SUCCESS; -} - -int32_t catalogGetDBVgInfo_a(CatalogParamWrapper* pCatalogWrapper, __async_cb_fn_t fp, void* param) { - CTG_API_ENTER(); - - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - SCatalogReqObj* pCatalogReq = taosMemoryCalloc(1, sizeof(SCatalogReqObj)); - pCatalogReq->userFp = fp; - pCatalogReq->pUserParam = param; - - // todo serialize the request message - pMsgSendInfo->msgInfo.pData = NULL; - pMsgSendInfo->msgInfo.len = 0; - - // set the callback and response struct. - pMsgSendInfo->msgType = TDMT_VND_FETCH; - pMsgSendInfo->fp = loadRemoteMetaCallback; - pMsgSendInfo->param = pCatalogReq; - - int32_t code = asyncSendMsgToServer(pCatalogWrapper->pTransporter, pCatalogWrapper->pMgmtEps, NULL, pMsgSendInfo); - - /// directly call the user function in case of failure - if (code != TSDB_CODE_SUCCESS) { - fp(NULL, param, code); - } -// CTG_API_LEAVE(code); -} int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo* dbInfo) { CTG_API_ENTER(); @@ -2791,7 +673,7 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - code = ctgPushUpdateVgMsgInQueue(pCtg, dbFName, dbId, dbInfo, false); + code = ctgPutUpdateVgToQueue(pCtg, dbFName, dbId, dbInfo, false); _return: @@ -2812,7 +694,7 @@ int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - CTG_ERR_JRET(ctgPushRmDBMsgInQueue(pCtg, dbFName, dbId)); + CTG_ERR_JRET(ctgPutRmDBToQueue(pCtg, dbFName, dbId)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); @@ -2825,7 +707,7 @@ int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, return 0; } -int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { +int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { CTG_API_ENTER(); int32_t code = 0; @@ -2838,30 +720,10 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - STableMeta *tblMeta = NULL; - bool inCache = false; - uint64_t dbId = 0; - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId)); - - if (!inCache) { - ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname); - goto _return; - } - - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); - - if (TSDB_SUPER_TABLE == tblMeta->tableType) { - CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, tblMeta->suid, true)); - } else { - CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, true)); - } + CTG_ERR_JRET(ctgRemoveTbMetaFromCache(pCtg, pTableName, true)); - _return: - - taosMemoryFreeClear(tblMeta); - + CTG_API_LEAVE(code); } @@ -2879,7 +741,7 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid, true)); + CTG_ERR_JRET(ctgPutRmStbToQueue(pCtg, dbFName, dbId, stbName, suid, true)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); @@ -2888,20 +750,24 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, const char *pIndexName, SIndexMeta** pIndexMeta) { - return 0; -} - int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_UNKNOWN_STB)); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)pTableName; + ctx.flag = CTG_FLAG_UNKNOWN_STB; + + CTG_API_LEAVE(ctgGetTbMeta(pCtg, pTrans, pMgmtEps, &ctx, pTableMeta)); } int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_STB)); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)pTableName; + ctx.flag = CTG_FLAG_STB; + + CTG_API_LEAVE(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, pTableMeta)); } int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { @@ -2928,7 +794,7 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta)); - CTG_ERR_JRET(ctgPushUpdateTblMsgInQueue(pCtg, output, false)); + CTG_ERR_JRET(ctgPutUpdateTbToQueue(pCtg, output, false)); CTG_API_LEAVE(code); @@ -2940,82 +806,6 @@ _return: CTG_API_LEAVE(code); } - - -int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver) { - *sver = -1; - - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); - return TSDB_CODE_SUCCESS; - } - - SCtgDBCache *dbCache = NULL; - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); - - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %s not in cache", pTableName->tname); - return TSDB_CODE_SUCCESS; - } - - int32_t tbType = 0; - uint64_t suid = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta* tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); - if (tbMeta) { - tbType = tbMeta->tableType; - suid = tbMeta->suid; - if (tbType != TSDB_CHILD_TABLE) { - *sver = tbMeta->sversion; - } - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; - } - - if (tbType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); - - return TSDB_CODE_SUCCESS; - } - - ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, suid); - - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &suid, sizeof(suid)); - if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("stb not in stbCache, suid:%"PRIx64, suid); - return TSDB_CODE_SUCCESS; - } - - if ((*stbMeta)->suid != suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, suid, (*stbMeta)->suid); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - *sver = (*stbMeta)->sversion; - - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); - - return TSDB_CODE_SUCCESS; -} - - int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { CTG_API_ENTER(); @@ -3029,12 +819,12 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm for (int32_t i = 0; i < tbNum; ++i) { STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - + if (CTG_IS_SYS_DBNAME(name.dbname)) { continue; } - ctgGetTbSverFromCache(pCtg, &name, &sver); + ctgReadTbSverFromCache(pCtg, &name, &sver); if (sver >= 0 && sver < pTb->sver) { catalogRemoveTableMeta(pCtg, &name); //TODO REMOVE STB FROM CACHE } @@ -3061,19 +851,27 @@ int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL, true)); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)pTableName; + ctx.flag = CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable); + + CTG_API_LEAVE(ctgRefreshTbMeta(CTG_PARAMS_LIST(), &ctx, NULL, true)); } int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable))); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)pTableName; + ctx.flag = CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable); + + CTG_API_LEAVE(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, pTableMeta)); } -int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { +int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3082,28 +880,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - int32_t code = 0; - - while (true) { - code = ctgGetTableDistVgInfo(pCtg, pRpc, pMgmtEps, pTableName, pVgList); - if (code) { - if (TSDB_CODE_CTG_VG_META_MISMATCH == code) { - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(CTG_FLAG_UNKNOWN_STB), NULL, true)); - - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); - CTG_ERR_JRET(ctgRefreshDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName)); - - continue; - } - } - - break; - } - -_return: - - CTG_API_LEAVE(code); + CTG_API_LEAVE(ctgGetTbDistVgInfo(pCtg, pTrans, pMgmtEps, (SName*)pTableName, pVgList)); } @@ -3151,8 +928,8 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, int32_t code = 0; pRsp->pTableMeta = NULL; - if (pReq->pTableName) { - int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName); + if (pReq->pTableMeta) { + int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableMeta); if (tbNum <= 0) { ctgError("empty table name list, tbNum:%d", tbNum); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); @@ -3165,10 +942,13 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } for (int32_t i = 0; i < tbNum; ++i) { - SName *name = taosArrayGet(pReq->pTableName, i); + SName *name = taosArrayGet(pReq->pTableMeta, i); STableMeta *pTableMeta = NULL; + SCtgTbMetaCtx ctx = {0}; + ctx.pName = name; + ctx.flag = CTG_FLAG_UNKNOWN_STB; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); + CTG_ERR_JRET(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, &pTableMeta)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); @@ -3180,7 +960,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (pReq->qNodeRequired) { pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeAddr)); - CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, pRsp->pQnodeList)); + CTG_ERR_JRET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), pRsp->pQnodeList, NULL)); } CTG_API_LEAVE(TSDB_CODE_SUCCESS); @@ -3201,15 +981,43 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { +int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == fp || NULL == param) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + SCtgJob *pJob = NULL; + CTG_ERR_JRET(ctgInitJob(CTG_PARAMS_LIST(), &pJob, reqId, pReq, fp, param)); + + CTG_ERR_JRET(ctgLaunchJob(pJob)); + + *jobId = pJob->refId; + +_return: + + if (pJob) { + taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); + + if (code) { + taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + } + } + + CTG_API_LEAVE(code); +} + +int32_t catalogGetQnodeList(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pQnodeList) { CTG_API_ENTER(); int32_t code = 0; - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pQnodeList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, pQnodeList)); + CTG_ERR_JRET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), pQnodeList, NULL)); _return: @@ -3265,59 +1073,50 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_ } -int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { +int32_t catalogGetDBCfg(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == dbFName || NULL == pDbCfg) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName || NULL == pDbCfg) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgGetDBCfgFromMnode(pCtg, pRpc, pMgmtEps, dbFName, pDbCfg)); + CTG_API_LEAVE(ctgGetDBCfgFromMnode(CTG_PARAMS_LIST(), dbFName, pDbCfg, NULL)); } -int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo) { +int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == indexName || NULL == pInfo) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == indexName || NULL == pInfo) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pRpc, pMgmtEps, indexName, pInfo)); + CTG_API_LEAVE(ctgGetIndexInfoFromMnode(CTG_PARAMS_LIST(), indexName, pInfo, NULL)); } -int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo) { +int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; - *pInfo = taosMemoryMalloc(sizeof(SFuncInfo)); - if (NULL == *pInfo) { - CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); - } - - CTG_ERR_JRET(ctgGetUdfInfoFromMnode(pCtg, pRpc, pMgmtEps, funcName, pInfo)); + CTG_ERR_JRET(ctgGetUdfInfoFromMnode(CTG_PARAMS_LIST(), funcName, pInfo, NULL)); _return: - - if (code) { - taosMemoryFreeClear(*pInfo); - } CTG_API_LEAVE(code); } -int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { +int32_t catalogChkAuth(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; - CTG_ERR_JRET(ctgChkAuth(pCtg, pRpc, pMgmtEps, user, dbFName, type, pass)); + CTG_ERR_JRET(ctgChkAuth(CTG_PARAMS_LIST(), user, dbFName, type, pass)); _return: @@ -3331,7 +1130,7 @@ int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgPushUpdateUserMsgInQueue(pCtg, pAuth, false)); + CTG_API_LEAVE(ctgPutUpdateUserToQueue(pCtg, pAuth, false)); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c new file mode 100644 index 0000000000..4908dc5101 --- /dev/null +++ b/source/libs/catalog/src/ctgAsync.c @@ -0,0 +1,1015 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" +#include "systable.h" +#include "tref.h" + +int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_TB_META; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbMetaCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbMetaCtx* ctx = pTask->taskCtx; + ctx->pName = taosMemoryMalloc(sizeof(*name)); + if (NULL == ctx->pName) { + taosMemoryFree(pTask->taskCtx); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(ctx->pName, name, sizeof(*name)); + ctx->flag = CTG_FLAG_UNKNOWN_STB; + + qDebug("QID:%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, pTask->type, name->tname); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_DB_VGROUP; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgDbVgCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgDbVgCtx* ctx = pTask->taskCtx; + + memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, pTask->type, dbFName); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetDbCfgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_DB_CFG; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgDbCfgCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgDbCfgCtx* ctx = pTask->taskCtx; + + memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, pTask->type, dbFName); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_TB_HASH; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbHashCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbHashCtx* ctx = pTask->taskCtx; + ctx->pName = taosMemoryMalloc(sizeof(*name)); + if (NULL == ctx->pName) { + taosMemoryFree(pTask->taskCtx); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(ctx->pName, name, sizeof(*name)); + tNameGetFullDbName(ctx->pName, ctx->dbFName); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, pTask->type, name->tname); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_QNODE; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + pTask->taskCtx = NULL; + + qDebug("QID:%" PRIx64 " task %d type %d initialized", pJob->queryId, taskIdx, pTask->type); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetIndexTask(SCtgJob *pJob, int32_t taskIdx, char *name) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_INDEX; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgIndexCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgIndexCtx* ctx = pTask->taskCtx; + + strcpy(ctx->indexFName, name); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, indexFName:%s", pJob->queryId, taskIdx, pTask->type, name); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetUdfTask(SCtgJob *pJob, int32_t taskIdx, char *name) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_UDF; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgUdfCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgUdfCtx* ctx = pTask->taskCtx; + + strcpy(ctx->udfName, name); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, udfName:%s", pJob->queryId, taskIdx, pTask->type, name); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, SUserAuthInfo *user) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskIdx); + + pTask->type = CTG_TASK_GET_USER; + pTask->taskId = taskIdx; + pTask->pJob = pJob; + + pTask->taskCtx = taosMemoryCalloc(1, sizeof(SCtgUserCtx)); + if (NULL == pTask->taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgUserCtx* ctx = pTask->taskCtx; + + memcpy(&ctx->user, user, sizeof(*user)); + + qDebug("QID:%" PRIx64 " task %d type %d initialized, user:%s", pJob->queryId, taskIdx, pTask->type, user->user); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param) { + int32_t code = 0; + int32_t tbMetaNum = (int32_t)taosArrayGetSize(pReq->pTableMeta); + int32_t dbVgNum = (int32_t)taosArrayGetSize(pReq->pDbVgroup); + int32_t tbHashNum = (int32_t)taosArrayGetSize(pReq->pTableHash); + int32_t udfNum = (int32_t)taosArrayGetSize(pReq->pUdf); + int32_t qnodeNum = pReq->qNodeRequired ? 1 : 0; + int32_t dbCfgNum = (int32_t)taosArrayGetSize(pReq->pDbCfg); + int32_t indexNum = (int32_t)taosArrayGetSize(pReq->pIndex); + int32_t userNum = (int32_t)taosArrayGetSize(pReq->pUser); + + int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum; + if (taskNum <= 0) { + ctgError("empty input for job, taskNum:%d", taskNum); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + *job = taosMemoryCalloc(1, sizeof(SCtgJob)); + if (NULL == *job) { + ctgError("calloc %d failed", (int32_t)sizeof(SCtgJob)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgJob *pJob = *job; + + pJob->queryId = reqId; + pJob->userFp = fp; + pJob->pCtg = pCtg; + pJob->pTrans = pTrans; + pJob->pMgmtEps = pMgmtEps; + pJob->userParam = param; + + pJob->tbMetaNum = tbMetaNum; + pJob->tbHashNum = tbHashNum; + pJob->qnodeNum = qnodeNum; + pJob->dbVgNum = dbVgNum; + pJob->udfNum = udfNum; + pJob->dbCfgNum = dbCfgNum; + pJob->indexNum = indexNum; + pJob->userNum = userNum; + + pJob->pTasks = taosArrayInit(taskNum, sizeof(SCtgTask)); + + if (NULL == pJob->pTasks) { + ctgError("taosArrayInit %d tasks failed", taskNum); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + int32_t taskIdx = 0; + for (int32_t i = 0; i < dbVgNum; ++i) { + char *dbFName = taosArrayGet(pReq->pDbVgroup, i); + CTG_ERR_JRET(ctgInitGetDbVgTask(pJob, taskIdx++, dbFName)); + } + + for (int32_t i = 0; i < dbCfgNum; ++i) { + char *dbFName = taosArrayGet(pReq->pDbCfg, i); + CTG_ERR_JRET(ctgInitGetDbCfgTask(pJob, taskIdx++, dbFName)); + } + + for (int32_t i = 0; i < tbMetaNum; ++i) { + SName *name = taosArrayGet(pReq->pTableMeta, i); + CTG_ERR_JRET(ctgInitGetTbMetaTask(pJob, taskIdx++, name)); + } + + for (int32_t i = 0; i < tbHashNum; ++i) { + SName *name = taosArrayGet(pReq->pTableHash, i); + CTG_ERR_JRET(ctgInitGetTbHashTask(pJob, taskIdx++, name)); + } + + for (int32_t i = 0; i < indexNum; ++i) { + char *indexName = taosArrayGet(pReq->pIndex, i); + CTG_ERR_JRET(ctgInitGetIndexTask(pJob, taskIdx++, indexName)); + } + + for (int32_t i = 0; i < udfNum; ++i) { + char *udfName = taosArrayGet(pReq->pUdf, i); + CTG_ERR_JRET(ctgInitGetUdfTask(pJob, taskIdx++, udfName)); + } + + for (int32_t i = 0; i < userNum; ++i) { + SUserAuthInfo *user = taosArrayGet(pReq->pUser, i); + CTG_ERR_JRET(ctgInitGetUserTask(pJob, taskIdx++, user)); + } + + if (qnodeNum) { + CTG_ERR_JRET(ctgInitGetQnodeTask(pJob, taskIdx++)); + } + + pJob->refId = taosAddRef(gCtgMgmt.jobPool, pJob); + if (pJob->refId < 0) { + ctgError("add job to ref failed, error: %s", tstrerror(terrno)); + CTG_ERR_JRET(terrno); + } + + taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); + + qDebug("QID:%" PRIx64 ", job %" PRIx64 " initialized, task num %d", pJob->queryId, pJob->refId, taskNum); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(*job); + + CTG_RET(code); +} + +int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pTableMeta) { + pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(STableMeta)); + if (NULL == pJob->jobRes.pTableMeta) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pTableMeta, pTask->res); + + taosMemoryFreeClear(pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpDbVgRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pDbVgroup) { + pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, POINTER_BYTES); + if (NULL == pJob->jobRes.pDbVgroup) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pDbVgroup, &pTask->res); + pTask->res = NULL; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpTbHashRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pTableHash) { + pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SVgroupInfo)); + if (NULL == pJob->jobRes.pTableHash) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pTableHash, &pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpIndexRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pIndex) { + pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SIndexInfo)); + if (NULL == pJob->jobRes.pIndex) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pIndex, pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpQnodeRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + + TSWAP(pJob->jobRes.pQnodeList, pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pDbCfg) { + pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SDbCfgInfo)); + if (NULL == pJob->jobRes.pDbCfg) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pDbCfg, &pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpUdfRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pUdfList) { + pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SFuncInfo)); + if (NULL == pJob->jobRes.pUdfList) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pUdfList, pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDumpUserRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pUser) { + pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(bool)); + if (NULL == pJob->jobRes.pUser) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pJob->jobRes.pUser, pTask->res); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { + SCtgJob* pJob = pTask->pJob; + int32_t code = 0; + + qDebug("QID:%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); + + if (rspCode) { + int32_t lastCode = atomic_val_compare_exchange_32(&pJob->rspCode, 0, rspCode); + if (0 == lastCode) { + CTG_ERR_JRET(rspCode); + } + + return TSDB_CODE_SUCCESS; + } + + int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); + if (taskDone < taosArrayGetSize(pJob->pTasks)) { + qDebug("task done: %d, total: %d", taskDone, (int32_t)taosArrayGetSize(pJob->pTasks)); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_JRET(ctgMakeAsyncRes(pJob)); + +_return: + + qDebug("QID:%" PRIx64 " user callback with rsp %s", pJob->queryId, tstrerror(code)); + + (*pJob->userFp)(&pJob->jobRes, pJob->userParam, code); + + taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + + CTG_RET(code); +} + +int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; + + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, ctx->pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + + CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgInfo, NULL, pTask)); + + return TSDB_CODE_SUCCESS; + } + case TDMT_MND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + + if (CTG_IS_META_NULL(pOut->metaType)) { + if (CTG_FLAG_IS_STB(ctx->flag)) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(ctx->pName, dbFName); + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + if (NULL != dbCache) { + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + + CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgInfo, NULL, pTask)); + + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask)); + } + + return TSDB_CODE_SUCCESS; + } + + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName)); + ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (pTask->msgCtx.lastOut) { + TSWAP(pTask->msgCtx.out, pTask->msgCtx.lastOut); + STableMetaOutput* pLastOut = (STableMetaOutput*)pTask->msgCtx.out; + TSWAP(pLastOut->tbMeta, pOut->tbMeta); + } + + break; + } + case TDMT_VND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + + if (CTG_IS_META_NULL(pOut->metaType)) { + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName)); + ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (CTG_FLAG_IS_STB(ctx->flag)) { + break; + } + + if (CTG_IS_META_TABLE(pOut->metaType) && TSDB_SUPER_TABLE == pOut->tbMeta->tableType) { + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName)); + + taosMemoryFreeClear(pOut->tbMeta); + + CTG_ERR_JRET(ctgGetTbMetaFromMnode(CTG_PARAMS_LIST(), ctx->pName, NULL, pTask)); + } else if (CTG_IS_META_BOTH(pOut->metaType)) { + int32_t exist = 0; + if (!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) { + CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pOut->dbFName, pOut->tbName, &exist)); + } + + if (0 == exist) { + TSWAP(pTask->msgCtx.lastOut, pTask->msgCtx.out); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), pOut->dbFName, pOut->tbName, NULL, pTask)); + } else { + taosMemoryFreeClear(pOut->tbMeta); + + SET_META_TYPE_CTABLE(pOut->metaType); + } + } + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + + ctgUpdateTbMetaToCache(pCtg, pOut, false); + + if (CTG_IS_META_BOTH(pOut->metaType)) { + memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + } else if (CTG_IS_META_CTABLE(pOut->metaType)) { + SName stbName = *ctx->pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = ctx->flag; + stbCtx.pName = &stbName; + + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (NULL == pOut->tbMeta) { + ctgDebug("stb no longer exist, stbName:%s", stbName.tname); + CTG_ERR_JRET(ctgRelaunchGetTbMetaTask(pTask)); + + return TSDB_CODE_SUCCESS; + } + + memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + } + + TSWAP(pTask->res, pOut->tbMeta); + +_return: + + if (dbCache) { + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetDbVgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; + + CTG_ERR_JRET(ctgGenerateVgList(pCtg, pOut->dbVgroup->vgHash, (SArray**)&pTask->res)); + + CTG_ERR_JRET(ctgPutUpdateVgToQueue(pCtg, ctx->dbFName, pOut->dbId, pOut->dbVgroup, false)); + pOut->dbVgroup = NULL; + + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetTbHashRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + SCtgTbHashCtx* ctx = (SCtgTbHashCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; + + pTask->res = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == pTask->res) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, ctx->pName, (SVgroupInfo*)pTask->res)); + + CTG_ERR_JRET(ctgPutUpdateVgToQueue(pCtg, ctx->dbFName, pOut->dbId, pOut->dbVgroup, false)); + pOut->dbVgroup = NULL; + + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetDbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetQnodeRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetUdfRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgHandleGetUserRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + bool pass = false; + SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pTask->msgCtx.out; + + if (pOut->superAuth) { + pass = true; + goto _return; + } + + if (pOut->createdDbs && taosHashGet(pOut->createdDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { + pass = true; + goto _return; + } + + if (ctx->user.type == AUTH_TYPE_READ && pOut->readDbs && taosHashGet(pOut->readDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { + pass = true; + } else if (ctx->user.type == AUTH_TYPE_WRITE && pOut->writeDbs && taosHashGet(pOut->writeDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { + pass = true; + } + +_return: + + if (TSDB_CODE_SUCCESS == code) { + pTask->res = taosMemoryCalloc(1, sizeof(bool)); + if (NULL == pTask->res) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + *(bool*)pTask->res = pass; + } + } + + ctgPutUpdateUserToQueue(pCtg, pOut, false); + pTask->msgCtx.out = NULL; + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + +int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + int32_t code = 0; + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + + if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { + ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(ctx->pName)); + + CTG_RET(ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), (char *)ctx->pName->dbname, (char *)ctx->pName->tname, NULL, pTask)); + } + + if (CTG_FLAG_IS_STB(ctx->flag)) { + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); + + // if get from mnode failed, will not try vnode + CTG_RET(ctgGetTbMetaFromMnode(CTG_PARAMS_LIST(), ctx->pName, NULL, pTask)); + } + + SCtgDBCache *dbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(ctx->pName, dbFName); + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + if (NULL == dbCache) { + SVgroupInfo vgInfo = {0}; + CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + + CTG_ERR_JRET(ctgGetTbMetaFromVnode(CTG_PARAMS_LIST(), ctx->pName, &vgInfo, NULL, pTask)); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask)); + } + +_return: + + if (dbCache) { + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + CTG_RET(code); +} + +int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + + CTG_ERR_RET(ctgGetTbMetaFromCache(CTG_PARAMS_LIST(), (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res)); + if (pTask->res) { + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { + int32_t code = 0; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgDBCache *dbCache = NULL; + SCtgDbVgCtx* pCtx = (SCtgDbVgCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); + if (NULL != dbCache) { + CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgInfo->vgHash, (SArray**)&pTask->res)); + + CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, pCtx->dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask)); + } + +_return: + + if (dbCache) { + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + CTG_RET(code); +} + +int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { + int32_t code = 0; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgDBCache *dbCache = NULL; + SCtgTbHashCtx* pCtx = (SCtgTbHashCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); + if (NULL != dbCache) { + pTask->res = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == pTask->res) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); + + CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, pCtx->dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pTrans, pMgmtEps, &input, NULL, pTask)); + } + +_return: + + if (dbCache) { + ctgReleaseVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + CTG_RET(code); +} + +int32_t ctgLaunchGetQnodeTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + + CTG_ERR_RET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetDbCfgTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgDbCfgCtx* pCtx = (SCtgDbCfgCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgGetDBCfgFromMnode(CTG_PARAMS_LIST(), pCtx->dbFName, NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetIndexTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgIndexCtx* pCtx = (SCtgIndexCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgGetIndexInfoFromMnode(CTG_PARAMS_LIST(), pCtx->indexFName, NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetUdfTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgUdfCtx* pCtx = (SCtgUdfCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgGetUdfInfoFromMnode(CTG_PARAMS_LIST(), pCtx->udfName, NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetUserTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = pTask->pJob->pMgmtEps; + SCtgUserCtx* pCtx = (SCtgUserCtx*)pTask->taskCtx; + bool inCache = false; + bool pass = false; + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pCtx->user.user, pCtx->user.dbFName, pCtx->user.type, &inCache, &pass)); + if (inCache) { + pTask->res = taosMemoryCalloc(1, sizeof(bool)); + if (NULL == pTask->res) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + *(bool*)pTask->res = pass; + + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(CTG_PARAMS_LIST(), pCtx->user.user, NULL, pTask)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask) { + ctgResetTbMetaTask(pTask); + + CTG_ERR_RET(ctgLaunchGetTbMetaTask(pTask)); + + return TSDB_CODE_SUCCESS; +} + +SCtgAsyncFps gCtgAsyncFps[] = { + {ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes}, + {ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes}, + {ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes}, + {ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes}, + {ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes}, + {ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes}, + {ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes}, + {ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes}, +}; + +int32_t ctgMakeAsyncRes(SCtgJob *pJob) { + int32_t code = 0; + int32_t taskNum = taosArrayGetSize(pJob->pTasks); + + for (int32_t i = 0; i < taskNum; ++i) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, i); + CTG_ERR_RET((*gCtgAsyncFps[pTask->type].dumpResFp)(pTask)); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgLaunchJob(SCtgJob *pJob) { + int32_t taskNum = taosArrayGetSize(pJob->pTasks); + + for (int32_t i = 0; i < taskNum; ++i) { + SCtgTask *pTask = taosArrayGet(pJob->pTasks, i); + + qDebug("QID:%" PRIx64 " start to launch task %d", pJob->queryId, pTask->taskId); + CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); + } + + return TSDB_CODE_SUCCESS; +} + + + diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c new file mode 100644 index 0000000000..f749e52a9e --- /dev/null +++ b/source/libs/catalog/src/ctgCache.c @@ -0,0 +1,1513 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" +#include "systable.h" + +SCtgAction gCtgAction[CTG_ACT_MAX] = { + { + CTG_ACT_UPDATE_VG, + "update vgInfo", + ctgActUpdateVg + }, + { + CTG_ACT_UPDATE_TBL, + "update tbMeta", + ctgActUpdateTb + }, + { + CTG_ACT_REMOVE_DB, + "remove DB", + ctgActRemoveDB + }, + { + CTG_ACT_REMOVE_STB, + "remove stbMeta", + ctgActRemoveStb + }, + { + CTG_ACT_REMOVE_TBL, + "remove tbMeta", + ctgActRemoveTb + }, + { + CTG_ACT_UPDATE_USER, + "update user", + ctgActUpdateUser + } +}; + + + + +int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { + CTG_LOCK(CTG_READ, &dbCache->vgLock); + + if (dbCache->deleted) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + + ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + + *inCache = false; + return TSDB_CODE_SUCCESS; + } + + + if (NULL == dbCache->vgInfo) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + + *inCache = false; + ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); + return TSDB_CODE_SUCCESS; + } + + *inCache = true; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + + if (dbCache->deleted) { + ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + return TSDB_CODE_SUCCESS; +} + +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { + taosHashRelease(pCtg->dbCache, dbCache); +} + +void ctgReleaseVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); +} + +void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); +} + + +int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_SYS_DBNAME(p + 1)) { + dbFName = p + 1; + } + + SCtgDBCache *dbCache = NULL; + if (acquire) { + dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); + } else { + dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); + } + + if (NULL == dbCache) { + *pCache = NULL; + ctgDebug("db not in cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + if (dbCache->deleted) { + if (acquire) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + ctgDebug("db is removing from cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); +} + +int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); +} + + +int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { + SCtgDBCache *dbCache = NULL; + + if (NULL == pCtg->dbCache) { + ctgDebug("empty db cache, dbFName:%s", dbFName); + goto _return; + } + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + bool inCache = false; + ctgAcquireVgInfo(pCtg, dbCache, &inCache); + if (!inCache) { + ctgDebug("vgInfo of db %s not in cache", dbFName); + goto _return; + } + + *pCache = dbCache; + + CTG_CACHE_STAT_ADD(vgHitNum, 1); + + ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); + + return TSDB_CODE_SUCCESS; + +_return: + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + + CTG_CACHE_STAT_ADD(vgMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { + if (NULL == pCtg->dbCache) { + *exist = 0; + ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); + return TSDB_CODE_SUCCESS; + } + + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + size_t sz = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == tbMeta) { + ctgReleaseDBCache(pCtg, dbCache); + + *exist = 0; + ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); + return TSDB_CODE_SUCCESS; + } + + *exist = 1; + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + + *pTableMeta = NULL; + + if (NULL == pCtg->dbCache) { + ctgDebug("empty tbmeta cache, tbName:%s", ctx->pName->tname); + return TSDB_CODE_SUCCESS; + } + + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { + strcpy(dbFName, ctx->pName->dbname); + } else { + tNameGetFullDbName(ctx->pName, dbFName); + } + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", ctx->pName->tname); + return TSDB_CODE_SUCCESS; + } + + int32_t sz = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + taosHashGetDup_m(dbCache->tbCache.metaCache, ctx->pName->tname, strlen(ctx->pName->tname), (void **)pTableMeta, &sz); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == *pTableMeta) { + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + return TSDB_CODE_SUCCESS; + } + + STableMeta* tbMeta = *pTableMeta; + ctx->tbInfo.inCache = true; + ctx->tbInfo.dbId = dbCache->dbId; + ctx->tbInfo.suid = tbMeta->suid; + ctx->tbInfo.tbType = tbMeta->tableType; + + if (tbMeta->tableType != TSDB_CHILD_TABLE) { + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, ctx->pName->tname); + + CTG_CACHE_STAT_ADD(tblHitNum, 1); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); + + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); + goto _return; + } + + if ((*stbMeta)->suid != tbMeta->suid) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + int32_t metaSize = CTG_META_SIZE(*stbMeta); + *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); + if (NULL == *pTableMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgError("realloc size[%d] failed", metaSize); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + + ctgReleaseDBCache(pCtg, dbCache); + + CTG_CACHE_STAT_ADD(tblHitNum, 1); + + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseDBCache(pCtg, dbCache); + taosMemoryFreeClear(*pTableMeta); + + CTG_CACHE_STAT_ADD(tblMissNum, 1); + + CTG_RET(code); +} + +int32_t ctgReadTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver) { + *sver = -1; + + if (NULL == pCtg->dbCache) { + ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + SCtgDBCache *dbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + int32_t tbType = 0; + uint64_t suid = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta* tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); + if (tbMeta) { + tbType = tbMeta->tableType; + suid = tbMeta->suid; + if (tbType != TSDB_CHILD_TABLE) { + *sver = tbMeta->sversion; + } + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == tbMeta) { + ctgReleaseDBCache(pCtg, dbCache); + return TSDB_CODE_SUCCESS; + } + + if (tbType != TSDB_CHILD_TABLE) { + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; + } + + ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, suid); + + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); + + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &suid, sizeof(suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("stb not in stbCache, suid:%"PRIx64, suid); + return TSDB_CODE_SUCCESS; + } + + if ((*stbMeta)->suid != suid) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, suid, (*stbMeta)->suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *sver = (*stbMeta)->sversion; + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { + if (NULL == pCtg->dbCache) { + ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); + return TSDB_CODE_SUCCESS; + } + + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName)); + + if (NULL == pTableMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); + ctgReleaseDBCache(pCtg, dbCache); + + return TSDB_CODE_SUCCESS; + } + + *tbType = atomic_load_8(&pTableMeta->tableType); + + taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { + if (NULL == pCtg->userCache) { + ctgDebug("empty user auth cache, user:%s", user); + goto _return; + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); + if (NULL == pUser) { + ctgDebug("user not in cache, user:%s", user); + goto _return; + } + + *inCache = true; + + ctgDebug("Got user from cache, user:%s", user); + CTG_CACHE_STAT_ADD(userHitNum, 1); + + if (pUser->superUser) { + *pass = true; + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &pUser->lock); + if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + CTG_UNLOCK(CTG_READ, &pUser->lock); + return TSDB_CODE_SUCCESS; + } + + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; + } + + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; + } + + CTG_UNLOCK(CTG_READ, &pUser->lock); + + return TSDB_CODE_SUCCESS; + +_return: + + *inCache = false; + CTG_CACHE_STAT_ADD(userMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + + +void ctgWaitAction(SCtgMetaAction *action) { + while (true) { + tsem_wait(&gCtgMgmt.queue.rspSem); + + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + tsem_post(&gCtgMgmt.queue.rspSem); + break; + } + + if (gCtgMgmt.queue.seqDone >= action->seqId) { + break; + } + + tsem_post(&gCtgMgmt.queue.rspSem); + sched_yield(); + } +} + +void ctgPopAction(SCtgMetaAction **action) { + SCtgQNode *orig = gCtgMgmt.queue.head; + + SCtgQNode *node = gCtgMgmt.queue.head->next; + gCtgMgmt.queue.head = gCtgMgmt.queue.head->next; + + CTG_QUEUE_SUB(); + + taosMemoryFreeClear(orig); + + *action = &node->action; +} + + +int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { + SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); + if (NULL == node) { + qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + CTG_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + action->seqId = atomic_add_fetch_64(&gCtgMgmt.queue.seqId, 1); + + node->action = *action; + + CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); + gCtgMgmt.queue.tail->next = node; + gCtgMgmt.queue.tail = node; + CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); + + CTG_QUEUE_ADD(); + CTG_RUNTIME_STAT_ADD(qNum, 1); + + tsem_post(&gCtgMgmt.queue.reqSem); + + ctgDebug("action [%s] added into queue", gCtgAction[action->act].name); + + if (action->syncReq) { + ctgWaitAction(action); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgPutRmDBToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; + SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_SYS_DBNAME(p + 1)) { + dbFName = p + 1; + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(action.data); + CTG_RET(code); +} + + +int32_t ctgPutRmStbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq}; + SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->stbName, stbName, sizeof(msg->stbName)); + msg->dbId = dbId; + msg->suid = suid; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(action.data); + CTG_RET(code); +} + + + +int32_t ctgPutRmTbToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq}; + SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->tbName, tbName, sizeof(msg->tbName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(action.data); + CTG_RET(code); +} + +int32_t ctgPutUpdateVgToQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; + SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); + ctgFreeVgInfo(dbInfo); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_SYS_DBNAME(p + 1)) { + dbFName = p + 1; + } + + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + msg->pCtg = pCtg; + msg->dbId = dbId; + msg->dbInfo = dbInfo; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgFreeVgInfo(dbInfo); + taosMemoryFreeClear(action.data); + CTG_RET(code); +} + +int32_t ctgPutUpdateTbToQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq}; + SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + char *p = strchr(output->dbFName, '.'); + if (p && CTG_IS_SYS_DBNAME(p + 1)) { + memmove(output->dbFName, p + 1, strlen(p + 1)); + } + + msg->pCtg = pCtg; + msg->output = output; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgPutUpdateUserToQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; + SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->userAuth = *pAuth; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + tFreeSGetUserAuthRsp(pAuth); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { + mgmt->slotRIdx = 0; + mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; + mgmt->type = type; + + size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; + + mgmt->slots = taosMemoryCalloc(1, msgSize); + if (NULL == mgmt->slots) { + qError("calloc %d failed", (int32_t)msgSize); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { + int16_t widx = abs((int)(id % mgmt->slotNum)); + + SCtgRentSlot *slot = &mgmt->slots[widx]; + int32_t code = 0; + + CTG_LOCK(CTG_WRITE, &slot->lock); + if (NULL == slot->meta) { + slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size); + if (NULL == slot->meta) { + qError("taosArrayInit %d failed, id:%"PRIx64", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + } + + if (NULL == taosArrayPush(slot->meta, meta)) { + qError("taosArrayPush meta to rent failed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + slot->needSort = true; + + qDebug("add meta to rent, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + +_return: + + CTG_UNLOCK(CTG_WRITE, &slot->lock); + CTG_RET(code); +} + +int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { + int16_t widx = abs((int)(id % mgmt->slotNum)); + + SCtgRentSlot *slot = &mgmt->slots[widx]; + int32_t code = 0; + + CTG_LOCK(CTG_WRITE, &slot->lock); + if (NULL == slot->meta) { + qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (slot->needSort) { + qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + taosArraySort(slot->meta, sortCompare); + slot->needSort = false; + qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + } + + void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); + if (NULL == orig) { + qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + memcpy(orig, meta, size); + + qDebug("meta in rent updated, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + +_return: + + CTG_UNLOCK(CTG_WRITE, &slot->lock); + + if (code) { + qWarn("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); + CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); + } + + CTG_RET(code); +} + +int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { + int16_t widx = abs((int)(id % mgmt->slotNum)); + + SCtgRentSlot *slot = &mgmt->slots[widx]; + int32_t code = 0; + + CTG_LOCK(CTG_WRITE, &slot->lock); + if (NULL == slot->meta) { + qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (slot->needSort) { + taosArraySort(slot->meta, sortCompare); + slot->needSort = false; + qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type); + } + + int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ); + if (idx < 0) { + qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + taosArrayRemove(slot->meta, idx); + + qDebug("meta in rent removed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + +_return: + + CTG_UNLOCK(CTG_WRITE, &slot->lock); + + CTG_RET(code); +} + + +int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { + int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1); + if (ridx >= mgmt->slotNum) { + ridx %= mgmt->slotNum; + atomic_store_16(&mgmt->slotRIdx, ridx); + } + + SCtgRentSlot *slot = &mgmt->slots[ridx]; + int32_t code = 0; + + CTG_LOCK(CTG_READ, &slot->lock); + if (NULL == slot->meta) { + qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type); + *num = 0; + goto _return; + } + + size_t metaNum = taosArrayGetSize(slot->meta); + if (metaNum <= 0) { + qDebug("no meta in slot:%d, type:%d", ridx, mgmt->type); + *num = 0; + goto _return; + } + + size_t msize = metaNum * size; + *res = taosMemoryMalloc(msize); + if (NULL == *res) { + qError("malloc %d failed", (int32_t)msize); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + void *meta = taosArrayGet(slot->meta, 0); + + memcpy(*res, meta, msize); + + *num = (uint32_t)metaNum; + + qDebug("Got %d meta from rent, type:%d", (int32_t)metaNum, mgmt->type); + +_return: + + CTG_UNLOCK(CTG_READ, &slot->lock); + + CTG_RET(code); +} + +int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { + while (true) { + int64_t msec = taosGetTimestampMs(); + int64_t lsec = atomic_load_64(&mgmt->lastReadMsec); + if ((msec - lsec) < CTG_RENT_SLOT_SECOND * 1000) { + *res = NULL; + *num = 0; + qDebug("too short time period to get expired meta, type:%d", mgmt->type); + return TSDB_CODE_SUCCESS; + } + + if (lsec != atomic_val_compare_exchange_64(&mgmt->lastReadMsec, lsec, msec)) { + continue; + } + + break; + } + + CTG_ERR_RET(ctgMetaRentGetImpl(mgmt, res, num, size)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { + int32_t code = 0; + + SCtgDBCache newDBCache = {0}; + newDBCache.dbId = dbId; + + newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache.metaCache) { + ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache.stbCache) { + ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache)); + if (code) { + if (HASH_NODE_EXIST(code)) { + ctgDebug("db already in cache, dbFName:%s", dbFName); + goto _return; + } + + ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + CTG_CACHE_STAT_ADD(dbNum, 1); + + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + + ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgFreeDbCache(&newDBCache); + + CTG_RET(code); +} + + +void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { + CTG_LOCK(CTG_WRITE, &cache->stbLock); + if (cache->stbCache) { + void *pIter = taosHashIterate(cache->stbCache, NULL); + while (pIter) { + uint64_t *suid = NULL; + suid = taosHashGetKey(pIter, NULL); + + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); + } + + pIter = taosHashIterate(cache->stbCache, pIter); + } + } + CTG_UNLOCK(CTG_WRITE, &cache->stbLock); +} + + +int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { + uint64_t dbId = dbCache->dbId; + + ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + atomic_store_8(&dbCache->deleted, 1); + + ctgRemoveStbRent(pCtg, &dbCache->tbCache); + + ctgFreeDbCache(dbCache); + + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + + ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { + ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + CTG_CACHE_STAT_SUB(dbNum, 1); + + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, dbFName, &dbCache); + + if (dbCache) { + // TODO OPEN IT +#if 0 + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#else + if (0 == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbId && (dbCache->dbId == 0)) { + dbCache->dbId = dbId; + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#endif + CTG_ERR_RET(ctgRemoveDBFromCache(pCtg, dbCache, dbFName)); + } + + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); + + ctgGetDBCache(pCtg, dbFName, &dbCache); + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgWriteDBVgInfoToCache(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { + int32_t code = 0; + SDBVgInfo* dbInfo = *pDbInfo; + + if (NULL == dbInfo->vgHash) { + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", + dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + bool newAdded = false; + SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; + + SCtgDBCache *dbCache = NULL; + CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SDBVgInfo *vgInfo = NULL; + CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + + if (dbCache->vgInfo) { + if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) { + ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + ctgWReleaseVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) { + ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); + ctgWReleaseVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + ctgFreeVgInfo(dbCache->vgInfo); + } + + dbCache->vgInfo = dbInfo; + + *pDbInfo = NULL; + + ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + + ctgWReleaseVgInfo(dbCache); + + dbCache = NULL; + + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + + CTG_RET(code); +} + + +int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { + SCtgTbMetaCache *tbCache = &dbCache->tbCache; + + CTG_LOCK(CTG_READ, &tbCache->metaLock); + if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + int8_t origType = 0; + uint64_t origSuid = 0; + bool isStb = meta->tableType == TSDB_SUPER_TABLE; + STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (orig) { + origType = orig->tableType; + + if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; + } + + if (origType == TSDB_SUPER_TABLE) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } else { + CTG_CACHE_STAT_SUB(stblNum, 1); + } + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); + + origSuid = orig->suid; + } + } + + if (isStb) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + } + + if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { + if (isStb) { + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + } + + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + if (NULL == orig) { + CTG_CACHE_STAT_ADD(tblNum, 1); + } + + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + ctgdShowTableMeta(pCtg, tbName, meta); + + if (!isStb) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + return TSDB_CODE_SUCCESS; + } + + STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + CTG_CACHE_STAT_ADD(stblNum, 1); + + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + + ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + + SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; + strcpy(metaRent.dbFName, dbFName); + strcpy(metaRent.stbName, tbName); + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq) { + STableMetaOutput* pOutput = NULL; + int32_t code = 0; + + CTG_ERR_RET(ctgCloneMetaOutput(pOut, &pOutput)); + CTG_ERR_JRET(ctgPutUpdateTbToQueue(pCtg, pOutput, syncReq)); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgFreeSTableMetaOutput(pOutput); + CTG_RET(code); +} + + +int32_t ctgActUpdateVg(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateVgMsg *msg = action->data; + + CTG_ERR_JRET(ctgWriteDBVgInfoToCache(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); + +_return: + + ctgFreeVgInfo(msg->dbInfo); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgActRemoveDB(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveDBMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + goto _return; + } + + if (dbCache->dbId != msg->dbId) { + ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); + goto _return; + } + + CTG_ERR_JRET(ctgRemoveDBFromCache(pCtg, dbCache, msg->dbFName)); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + +int32_t ctgActUpdateTb(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateTblMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + STableMetaOutput* output = msg->output; + SCtgDBCache *dbCache = NULL; + + if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { + ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { + int32_t metaSize = CTG_META_SIZE(output->tbMeta); + + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + } + + if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + } + +_return: + + if (output) { + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); + } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + +int32_t ctgActRemoveStb(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveStbMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + if (msg->dbId && (dbCache->dbId != msg->dbId)) { + ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { + ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + } else { + CTG_CACHE_STAT_SUB(stblNum, 1); + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + } else { + CTG_CACHE_STAT_SUB(tblNum, 1); + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + + ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); + + ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgActRemoveTb(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveTblMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId != msg->dbId) { + ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } else { + CTG_CACHE_STAT_SUB(tblNum, 1); + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgActUpdateUser(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateUserMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + if (NULL == pCtg->userCache) { + pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pCtg->userCache) { + ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); + if (NULL == pUser) { + SCtgUserAuth userAuth = {0}; + + userAuth.version = msg->userAuth.version; + userAuth.superUser = msg->userAuth.superAuth; + userAuth.createdDbs = msg->userAuth.createdDbs; + userAuth.readDbs = msg->userAuth.readDbs; + userAuth.writeDbs = msg->userAuth.writeDbs; + + if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { + ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + taosMemoryFreeClear(msg); + + return TSDB_CODE_SUCCESS; + } + + pUser->version = msg->userAuth.version; + + CTG_LOCK(CTG_WRITE, &pUser->lock); + + taosHashCleanup(pUser->createdDbs); + pUser->createdDbs = msg->userAuth.createdDbs; + msg->userAuth.createdDbs = NULL; + + taosHashCleanup(pUser->readDbs); + pUser->readDbs = msg->userAuth.readDbs; + msg->userAuth.readDbs = NULL; + + taosHashCleanup(pUser->writeDbs); + pUser->writeDbs = msg->userAuth.writeDbs; + msg->userAuth.writeDbs = NULL; + + CTG_UNLOCK(CTG_WRITE, &pUser->lock); + +_return: + + + taosHashCleanup(msg->userAuth.createdDbs); + taosHashCleanup(msg->userAuth.readDbs); + taosHashCleanup(msg->userAuth.writeDbs); + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + +void* ctgUpdateThreadFunc(void* param) { + setThreadName("catalog"); + + qInfo("catalog update thread started"); + + CTG_LOCK(CTG_READ, &gCtgMgmt.lock); + + while (true) { + if (tsem_wait(&gCtgMgmt.queue.reqSem)) { + qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } + + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + tsem_post(&gCtgMgmt.queue.rspSem); + break; + } + + SCtgMetaAction *action = NULL; + ctgPopAction(&action); + SCatalog *pCtg = ((SCtgUpdateMsgHeader *)action->data)->pCtg; + + ctgDebug("process [%s] action", gCtgAction[action->act].name); + + (*gCtgAction[action->act].func)(action); + + gCtgMgmt.queue.seqDone = action->seqId; + + if (action->syncReq) { + tsem_post(&gCtgMgmt.queue.rspSem); + } + + CTG_RUNTIME_STAT_ADD(qDoneNum, 1); + + ctgdShowClusterCache(pCtg); + } + + CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); + + qInfo("catalog update thread stopped"); + + return NULL; +} + + +int32_t ctgStartUpdateThread() { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + CTG_ERR_RET(terrno); + } + + taosThreadAttrDestroy(&thAttr); + return TSDB_CODE_SUCCESS; +} + + + diff --git a/source/libs/catalog/src/catalogDbg.c b/source/libs/catalog/src/ctgDbg.c similarity index 100% rename from source/libs/catalog/src/catalogDbg.c rename to source/libs/catalog/src/ctgDbg.c diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c new file mode 100644 index 0000000000..ee5644ec6d --- /dev/null +++ b/source/libs/catalog/src/ctgRemote.c @@ -0,0 +1,582 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" +#include "systable.h" +#include "ctgRemote.h" +#include "tref.h" + +int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target) { + int32_t code = 0; + + switch (reqType) { + case TDMT_MND_QNODE_LIST: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for qnode list, error:%s", tstrerror(rspCode)); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process qnode list rsp failed, error:%s", tstrerror(rspCode)); + CTG_ERR_RET(code); + } + + qDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out)); + break; + } + case TDMT_MND_USE_DB: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for use db, error:%s, dbFName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process use db rsp failed, error:%s, dbFName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got db vgInfo from mnode, dbFName:%s", target); + break; + } + case TDMT_MND_GET_DB_CFG: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for get db cfg, error:%s, db:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process get db cfg rsp failed, error:%s, db:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got db cfg from mnode, dbFName:%s", target); + break; + } + case TDMT_MND_GET_INDEX: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for get index, error:%s, indexName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process get index rsp failed, error:%s, indexName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got index from mnode, indexName:%s", target); + break; + } + case TDMT_MND_RETRIEVE_FUNC: { + if (TSDB_CODE_SUCCESS != rspCode) { + if (TSDB_CODE_MND_FUNC_NOT_EXIST == rspCode) { + qError("funcName %s not exist in mnode", target); + CTG_RET(TSDB_CODE_SUCCESS); + } + + qError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process get udf rsp failed, error:%s, funcName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got udf from mnode, funcName:%s", target); + break; + } + case TDMT_MND_GET_USER_AUTH: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for get user auth, error:%s, user:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process get user auth rsp failed, error:%s, user:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got user auth from mnode, user:%s", target); + break; + } + case TDMT_MND_TABLE_META: { + if (TSDB_CODE_SUCCESS != rspCode) { + if (CTG_TABLE_NOT_EXIST(rspCode)) { + SET_META_TYPE_NULL(((STableMetaOutput*)out)->metaType); + qDebug("stablemeta not exist in mnode, tbFName:%s", target); + return TSDB_CODE_SUCCESS; + } + + qError("error rsp for stablemeta from mnode, error:%s, tbFName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process mnode stablemeta rsp failed, error:%s, tbFName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got table meta from mnode, tbFName:%s", target); + break; + } + case TDMT_VND_TABLE_META: { + if (TSDB_CODE_SUCCESS != rspCode) { + if (CTG_TABLE_NOT_EXIST(rspCode)) { + SET_META_TYPE_NULL(((STableMetaOutput*)out)->metaType); + qDebug("tablemeta not exist in vnode, tbFName:%s", target); + return TSDB_CODE_SUCCESS; + } + + qError("error rsp for table meta from vnode, code:%s, tbFName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process vnode tablemeta rsp failed, code:%s, tbFName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got table meta from vnode, tbFName:%s", target); + break; + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgHandleMsgCallback(void *param, const SDataBuf *pMsg, int32_t rspCode) { + SCtgTaskCallbackParam* cbParam = (SCtgTaskCallbackParam*)param; + int32_t code = 0; + + CTG_API_ENTER(); + + SCtgJob* pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId); + if (NULL == pJob) { + qDebug("job refId %" PRIx64 " already dropped", cbParam->refId); + goto _return; + } + + SCtgTask *pTask = taosArrayGet(pJob->pTasks, cbParam->taskId); + + qDebug("QID:%" PRIx64 " task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); + + CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode)); + +_return: + + if (pJob) { + taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); + } + + taosMemoryFree(param); + + CTG_API_LEAVE(code); +} + + +int32_t ctgMakeMsgSendInfo(SCtgTask* pTask, int32_t msgType, SMsgSendInfo **pMsgSendInfo) { + int32_t code = 0; + SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == msgSendInfo) { + qError("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SCtgTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam)); + if (NULL == param) { + qError("calloc %d failed", (int32_t)sizeof(SCtgTaskCallbackParam)); + CTG_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param->reqType = msgType; + param->queryId = pTask->pJob->queryId; + param->refId = pTask->pJob->refId; + param->taskId = pTask->taskId; + + msgSendInfo->param = param; + msgSendInfo->fp = ctgHandleMsgCallback; + + *pMsgSendInfo = msgSendInfo; + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFree(param); + taosMemoryFree(msgSendInfo); + + CTG_RET(code); +} + +int32_t ctgAsyncSendMsg(CTG_PARAMS, SCtgTask* pTask, int32_t msgType, void *msg, uint32_t msgSize) { + int32_t code = 0; + SMsgSendInfo *pMsgSendInfo = NULL; + CTG_ERR_JRET(ctgMakeMsgSendInfo(pTask, msgType, &pMsgSendInfo)); + + pMsgSendInfo->msgInfo.pData = msg; + pMsgSendInfo->msgInfo.len = msgSize; + pMsgSendInfo->msgInfo.handle = NULL; + pMsgSendInfo->msgType = msgType; + + int64_t transporterId = 0; + code = asyncSendMsgToServer(pTrans, (SEpSet*)pMgmtEps, &transporterId, pMsgSendInfo); + if (code) { + ctgError("asyncSendMsgToSever failed, error: %s", tstrerror(code)); + CTG_ERR_JRET(code); + } + + ctgDebug("req msg sent, reqId:%" PRIx64 ", msg type:%d, %s", pTask->pJob->queryId, msgType, TMSG_INFO(msgType)); + return TSDB_CODE_SUCCESS; + +_return: + + if (pMsgSendInfo) { + taosMemoryFreeClear(pMsgSendInfo->param); + taosMemoryFreeClear(pMsgSendInfo); + } + + CTG_RET(code); +} + + + + +int32_t ctgGetQnodeListFromMnode(CTG_PARAMS, SArray *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_QNODE_LIST; + + ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pMgmtEps->inUse); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](NULL, &msg, 0, &msgLen); + if (code) { + ctgError("Build qnode list msg failed, error:%s", tstrerror(code)); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, NULL)); + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetDBVgInfoFromMnode(CTG_PARAMS, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_USE_DB; + + ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](input, &msg, 0, &msgLen); + if (code) { + ctgError("Build use db msg failed, code:%x, db:%s", code, input->db); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(SUseDbOutput)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, input->db)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, input->db)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetDBCfgFromMnode(CTG_PARAMS, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_GET_DB_CFG; + + ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)dbFName, &msg, 0, &msgLen); + if (code) { + ctgError("Build get db cfg msg failed, code:%x, db:%s", code, dbFName); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(SDbCfgInfo)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)dbFName)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_GET_DB_CFG, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)dbFName)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetIndexInfoFromMnode(CTG_PARAMS, const char *indexName, SIndexInfo *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_GET_INDEX; + + ctgDebug("try to get index from mnode, indexName:%s", indexName); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)indexName, &msg, 0, &msgLen); + if (code) { + ctgError("Build get index msg failed, code:%x, db:%s", code, indexName); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(SIndexInfo)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)indexName)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)indexName)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetUdfInfoFromMnode(CTG_PARAMS, const char *funcName, SFuncInfo *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_RETRIEVE_FUNC; + + ctgDebug("try to get udf info from mnode, funcName:%s", funcName); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)funcName, &msg, 0, &msgLen); + if (code) { + ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(SFuncInfo)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)funcName)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)funcName)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetUserDbAuthFromMnode(CTG_PARAMS, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_GET_USER_AUTH; + + ctgDebug("try to get user auth from mnode, user:%s", user); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void *)user, &msg, 0, &msgLen); + if (code) { + ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(SGetUserAuthRsp)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)user)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)user)); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetTbMetaFromMnodeImpl(CTG_PARAMS, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask) { + SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_TABLE_META; + char tbFName[TSDB_TABLE_FNAME_LEN]; + sprintf(tbFName, "%s.%s", dbFName, tbName); + + ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen); + if (code) { + ctgError("Build mnode stablemeta msg failed, code:%x", code); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(STableMetaOutput)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetTbMetaFromMnode(CTG_PARAMS, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask) { + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); + + return ctgGetTbMetaFromMnodeImpl(CTG_PARAMS_LIST(), dbFName, (char *)pTableName->tname, out, pTask); +} + +int32_t ctgGetTbMetaFromVnode(CTG_PARAMS, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask) { + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); + int32_t reqType = TDMT_VND_TABLE_META; + char tbFName[TSDB_TABLE_FNAME_LEN]; + sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); + + ctgDebug("try to get table meta from vnode, vgId:%d, tbFName:%s", vgroupInfo->vgId, tbFName); + + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; + char *msg = NULL; + int32_t msgLen = 0; + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen); + if (code) { + ctgError("Build vnode tablemeta msg failed, code:%x, tbFName:%s", code, tbFName); + CTG_ERR_RET(code); + } + + if (pTask) { + void* pOut = taosMemoryCalloc(1, sizeof(STableMetaOutput)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); + + CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c new file mode 100644 index 0000000000..2d7fb8aa97 --- /dev/null +++ b/source/libs/catalog/src/ctgUtil.c @@ -0,0 +1,577 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" +#include "systable.h" + +void ctgFreeSMetaData(SMetaData* pData) { + taosArrayDestroy(pData->pTableMeta); + pData->pTableMeta = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pData->pDbVgroup); ++i) { + SArray** pArray = taosArrayGet(pData->pDbVgroup, i); + taosArrayDestroy(*pArray); + } + taosArrayDestroy(pData->pDbVgroup); + pData->pDbVgroup = NULL; + + taosArrayDestroy(pData->pTableHash); + pData->pTableHash = NULL; + + taosArrayDestroy(pData->pUdfList); + pData->pUdfList = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pData->pDbCfg); ++i) { + SDbCfgInfo* pInfo = taosArrayGet(pData->pDbCfg, i); + taosArrayDestroy(pInfo->pRetensions); + } + taosArrayDestroy(pData->pDbCfg); + pData->pDbCfg = NULL; + + taosArrayDestroy(pData->pIndex); + pData->pIndex = NULL; + + taosArrayDestroy(pData->pUser); + pData->pUser = NULL; + + taosArrayDestroy(pData->pQnodeList); + pData->pQnodeList = NULL; +} + +void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) { + taosHashCleanup(userCache->createdDbs); + taosHashCleanup(userCache->readDbs); + taosHashCleanup(userCache->writeDbs); +} + +void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { + if (NULL == mgmt->slots) { + return; + } + + for (int32_t i = 0; i < mgmt->slotNum; ++i) { + SCtgRentSlot *slot = &mgmt->slots[i]; + if (slot->meta) { + taosArrayDestroy(slot->meta); + slot->meta = NULL; + } + } + + taosMemoryFreeClear(mgmt->slots); +} + + +void ctgFreeTbMetaCache(SCtgTbMetaCache *cache) { + CTG_LOCK(CTG_WRITE, &cache->stbLock); + if (cache->stbCache) { + int32_t stblNum = taosHashGetSize(cache->stbCache); + taosHashCleanup(cache->stbCache); + cache->stbCache = NULL; + CTG_CACHE_STAT_SUB(stblNum, stblNum); + } + CTG_UNLOCK(CTG_WRITE, &cache->stbLock); + + CTG_LOCK(CTG_WRITE, &cache->metaLock); + if (cache->metaCache) { + int32_t tblNum = taosHashGetSize(cache->metaCache); + taosHashCleanup(cache->metaCache); + cache->metaCache = NULL; + CTG_CACHE_STAT_SUB(tblNum, tblNum); + } + CTG_UNLOCK(CTG_WRITE, &cache->metaLock); +} + +void ctgFreeVgInfo(SDBVgInfo *vgInfo) { + if (NULL == vgInfo) { + return; + } + + if (vgInfo->vgHash) { + taosHashCleanup(vgInfo->vgHash); + vgInfo->vgHash = NULL; + } + + taosMemoryFreeClear(vgInfo); +} + +void ctgFreeDbCache(SCtgDBCache *dbCache) { + if (NULL == dbCache) { + return; + } + + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + ctgFreeVgInfo (dbCache->vgInfo); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + + ctgFreeTbMetaCache(&dbCache->tbCache); +} + + +void ctgFreeHandle(SCatalog* pCtg) { + ctgFreeMetaRent(&pCtg->dbRent); + ctgFreeMetaRent(&pCtg->stbRent); + + if (pCtg->dbCache) { + int32_t dbNum = taosHashGetSize(pCtg->dbCache); + + void *pIter = taosHashIterate(pCtg->dbCache, NULL); + while (pIter) { + SCtgDBCache *dbCache = pIter; + + atomic_store_8(&dbCache->deleted, 1); + + ctgFreeDbCache(dbCache); + + pIter = taosHashIterate(pCtg->dbCache, pIter); + } + + taosHashCleanup(pCtg->dbCache); + + CTG_CACHE_STAT_SUB(dbNum, dbNum); + } + + if (pCtg->userCache) { + int32_t userNum = taosHashGetSize(pCtg->userCache); + + void *pIter = taosHashIterate(pCtg->userCache, NULL); + while (pIter) { + SCtgUserAuth *userCache = pIter; + + ctgFreeSCtgUserAuth(userCache); + + pIter = taosHashIterate(pCtg->userCache, pIter); + } + + taosHashCleanup(pCtg->userCache); + + CTG_CACHE_STAT_SUB(userNum, userNum); + } + + taosMemoryFree(pCtg); +} + + +void ctgFreeSUseDbOutput(SUseDbOutput* pOutput) { + if (NULL == pOutput || NULL == pOutput->dbVgroup) { + return; + } + + taosHashCleanup(pOutput->dbVgroup->vgHash); + taosMemoryFreeClear(pOutput->dbVgroup); + taosMemoryFree(pOutput); +} + +void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { + taosMemoryFreeClear(pCtx->target); + if (NULL == pCtx->out) { + return; + } + + switch (pCtx->reqType) { + case TDMT_MND_GET_DB_CFG: { + SDbCfgInfo* pOut = (SDbCfgInfo*)pCtx->out; + taosArrayDestroy(pOut->pRetensions); + taosMemoryFreeClear(pCtx->out); + break; + } + case TDMT_MND_USE_DB:{ + SUseDbOutput* pOut = (SUseDbOutput*)pCtx->out; + ctgFreeSUseDbOutput(pOut); + pCtx->out = NULL; + break; + } + case TDMT_MND_GET_INDEX: { + SIndexInfo* pOut = (SIndexInfo*)pCtx->out; + taosMemoryFreeClear(pCtx->out); + break; + } + case TDMT_MND_QNODE_LIST: { + SArray* pOut = (SArray*)pCtx->out; + taosArrayDestroy(pOut); + pCtx->out = NULL; + break; + } + case TDMT_VND_TABLE_META: + case TDMT_MND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pCtx->out; + taosMemoryFree(pOut->tbMeta); + taosMemoryFreeClear(pCtx->out); + break; + } + case TDMT_MND_RETRIEVE_FUNC: { + SFuncInfo* pOut = (SFuncInfo*)pCtx->out; + taosMemoryFree(pOut->pCode); + taosMemoryFree(pOut->pComment); + taosMemoryFreeClear(pCtx->out); + break; + } + case TDMT_MND_GET_USER_AUTH: { + SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pCtx->out; + taosHashCleanup(pOut->createdDbs); + taosHashCleanup(pOut->readDbs); + taosHashCleanup(pOut->writeDbs); + taosMemoryFreeClear(pCtx->out); + break; + } + default: + qError("invalid reqType %d", pCtx->reqType); + break; + } +} + +void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput) { + if (NULL == pOutput) { + return; + } + + taosMemoryFree(pOutput->tbMeta); + taosMemoryFree(pOutput); +} + + +void ctgResetTbMetaTask(SCtgTask* pTask) { + SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx; + memset(&taskCtx->tbInfo, 0, sizeof(taskCtx->tbInfo)); + taskCtx->flag = CTG_FLAG_UNKNOWN_STB; + + if (pTask->msgCtx.lastOut) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut); + pTask->msgCtx.lastOut = NULL; + } + if (pTask->msgCtx.out) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.out); + pTask->msgCtx.out = NULL; + } + taosMemoryFreeClear(pTask->msgCtx.target); + taosMemoryFreeClear(pTask->res); +} + +void ctgFreeTask(SCtgTask* pTask) { + ctgFreeMsgCtx(&pTask->msgCtx); + + switch (pTask->type) { + case CTG_TASK_GET_QNODE: { + taosArrayDestroy((SArray*)pTask->res); + pTask->res = NULL; + break; + } + case CTG_TASK_GET_TB_META: { + SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx; + taosMemoryFreeClear(taskCtx->pName); + if (pTask->msgCtx.lastOut) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut); + pTask->msgCtx.lastOut = NULL; + } + taosMemoryFreeClear(pTask->res); + break; + } + case CTG_TASK_GET_DB_VGROUP: { + taosArrayDestroy((SArray*)pTask->res); + pTask->res = NULL; + break; + } + case CTG_TASK_GET_DB_CFG: { + if (pTask->res) { + taosArrayDestroy(((SDbCfgInfo*)pTask->res)->pRetensions); + taosMemoryFreeClear(pTask->res); + } + break; + } + case CTG_TASK_GET_TB_HASH: { + SCtgTbHashCtx* taskCtx = (SCtgTbHashCtx*)pTask->taskCtx; + taosMemoryFreeClear(taskCtx->pName); + taosMemoryFreeClear(pTask->res); + break; + } + case CTG_TASK_GET_INDEX: { + taosMemoryFreeClear(pTask->taskCtx); + taosMemoryFreeClear(pTask->res); + break; + } + case CTG_TASK_GET_UDF: { + taosMemoryFreeClear(pTask->taskCtx); + taosMemoryFreeClear(pTask->res); + break; + } + case CTG_TASK_GET_USER: { + taosMemoryFreeClear(pTask->taskCtx); + taosMemoryFreeClear(pTask->res); + break; + } + default: + qError("invalid task type %d", pTask->type); + break; + } +} + +void ctgFreeTasks(SArray* pArray) { + if (NULL == pArray) { + return; + } + + int32_t num = taosArrayGetSize(pArray); + for (int32_t i = 0; i < num; ++i) { + SCtgTask* pTask = taosArrayGet(pArray, i); + ctgFreeTask(pTask); + } + + taosArrayDestroy(pArray); +} + +void ctgFreeJob(void* job) { + if (NULL == job) { + return; + } + + SCtgJob* pJob = (SCtgJob*)job; + + int64_t rid = pJob->refId; + uint64_t qid = pJob->queryId; + + ctgFreeTasks(pJob->pTasks); + + ctgFreeSMetaData(&pJob->jobRes); + + taosMemoryFree(job); + + qDebug("QID:%" PRIx64 ", job %" PRIx64 " freed", qid, rid); +} + +int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target) { + ctgFreeMsgCtx(pCtx); + + pCtx->reqType = reqType; + pCtx->out = out; + if (target) { + pCtx->target = strdup(target); + if (NULL == pCtx->target) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } else { + pCtx->target = NULL; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { + switch (hashMethod) { + default: + *fp = MurmurHash3_32; + break; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) { + SHashObj *vgroupHash = NULL; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; + int32_t code = 0; + int32_t vgNum = taosHashGetSize(vgHash); + + vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); + if (NULL == vgList) { + ctgError("taosArrayInit failed, num:%d", vgNum); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + void *pIter = taosHashIterate(vgHash, NULL); + while (pIter) { + vgInfo = pIter; + + if (NULL == taosArrayPush(vgList, vgInfo)) { + ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); + taosHashCancelIterate(vgHash, pIter); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + pIter = taosHashIterate(vgHash, pIter); + vgInfo = NULL; + } + + *pList = vgList; + + ctgDebug("Got vgList from cache, vgNum:%d", vgNum); + + return TSDB_CODE_SUCCESS; + +_return: + + if (vgList) { + taosArrayDestroy(vgList); + } + + CTG_RET(code); +} + + +int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { + int32_t code = 0; + + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, db); + + if (vgNum <= 0) { + ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + tableNameHashFp fp = NULL; + SVgroupInfo *vgInfo = NULL; + + CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pTableName, tbFullName); + + uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); + + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + vgInfo = pIter; + if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { + taosHashCancelIterate(dbInfo->vgHash, pIter); + break; + } + + pIter = taosHashIterate(dbInfo->vgHash, pIter); + vgInfo = NULL; + } + + if (NULL == vgInfo) { + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *pVgroup = *vgInfo; + + CTG_RET(code); +} + +int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { + if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { + return -1; + } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { + return 1; + } else { + return 0; + } +} + +int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { + if (*(int64_t *)key1 < ((SDbVgVersion*)key2)->dbId) { + return -1; + } else if (*(int64_t *)key1 > ((SDbVgVersion*)key2)->dbId) { + return 1; + } else { + return 0; + } +} + +int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { + if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { + return -1; + } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + return 1; + } else { + return 0; + } +} + +int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) { + if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { + return -1; + } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { + return 1; + } else { + return 0; + } +} + + + + +int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { + *dst = taosMemoryMalloc(sizeof(SDBVgInfo)); + if (NULL == *dst) { + qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(*dst, src, sizeof(SDBVgInfo)); + + size_t hashSize = taosHashGetSize(src->vgHash); + (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == (*dst)->vgHash) { + qError("taosHashInit %d failed", (int32_t)hashSize); + taosMemoryFreeClear(*dst); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + int32_t *vgId = NULL; + void *pIter = taosHashIterate(src->vgHash, NULL); + while (pIter) { + vgId = taosHashGetKey(pIter, NULL); + + if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) { + qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); + taosHashCancelIterate(src->vgHash, pIter); + taosHashCleanup((*dst)->vgHash); + taosMemoryFreeClear(*dst); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + pIter = taosHashIterate(src->vgHash, pIter); + } + + + return TSDB_CODE_SUCCESS; +} + + + +int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { + *pOutput = taosMemoryMalloc(sizeof(STableMetaOutput)); + if (NULL == *pOutput) { + qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(*pOutput, output, sizeof(STableMetaOutput)); + + if (output->tbMeta) { + int32_t metaSize = CTG_META_SIZE(output->tbMeta); + (*pOutput)->tbMeta = taosMemoryMalloc(metaSize); + if (NULL == (*pOutput)->tbMeta) { + qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + taosMemoryFreeClear(*pOutput); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize); + } + + return TSDB_CODE_SUCCESS; +} + + + diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index cff0087d6c..6c7d1ac4ca 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -40,10 +40,8 @@ namespace { -extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, - bool *inCache, int32_t flag, uint64_t *dbId); extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); -extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); +extern "C" int32_t ctgActUpdateTb(SCtgMetaAction *action); extern "C" int32_t ctgdEnableDebug(char *option); extern "C" int32_t ctgdGetStatNum(char *option, void *res); @@ -52,7 +50,7 @@ void ctgTestSetRspCTableMeta(); void ctgTestSetRspSTableMeta(); void ctgTestSetRspMultiSTableMeta(); -extern "C" SCatalogMgmt gCtgMgmt; +//extern "C" SCatalogMgmt gCtgMgmt; enum { CTGT_RSP_VGINFO = 1, @@ -859,8 +857,12 @@ void *ctgTestGetCtableMetaThread(void *param) { strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); + SCtgTbMetaCtx ctx = {0}; + ctx.pName = &cn; + ctx.flag = CTG_FLAG_UNKNOWN_STB; + while (!ctgTestStop) { - code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &inCache, 0, NULL); + code = ctgReadTbMetaFromCache(pCtg, &ctx, &tbMeta); if (code || !inCache) { assert(0); } @@ -899,7 +901,7 @@ void *ctgTestSetCtableMetaThread(void *param) { msg->output = output; action.data = msg; - code = ctgActUpdateTbl(&action); + code = ctgActUpdateTb(&action); if (code) { assert(0); } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 3b1e66f2ad..80e777196e 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -66,22 +66,18 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { } static int32_t getUdfInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) { - SFuncInfo* pInfo = NULL; - int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &pInfo); + SFuncInfo funcInfo = {0}; + int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &funcInfo); if (TSDB_CODE_SUCCESS != code) { return code; } - if (NULL == pInfo) { - snprintf(pParam->pErrBuf, pParam->errBufLen, "Invalid function name: %s", pFunc->functionName); - return TSDB_CODE_FUNC_INVALID_FUNTION; - } + pFunc->funcType = FUNCTION_TYPE_UDF; - pFunc->funcId = TSDB_FUNC_TYPE_AGGREGATE == pInfo->funcType ? FUNC_AGGREGATE_UDF_ID : FUNC_SCALAR_UDF_ID; - pFunc->node.resType.type = pInfo->outputType; - pFunc->node.resType.bytes = pInfo->outputLen; - pFunc->udfBufSize = pInfo->bufSize; - tFreeSFuncInfo(pInfo); - taosMemoryFree(pInfo); + pFunc->funcId = TSDB_FUNC_TYPE_AGGREGATE == funcInfo.funcType ? FUNC_AGGREGATE_UDF_ID : FUNC_SCALAR_UDF_ID; + pFunc->node.resType.type = funcInfo.outputType; + pFunc->node.resType.bytes = funcInfo.outputLen; + pFunc->udfBufSize = funcInfo.bufSize; + tFreeSFuncInfo(&funcInfo); return TSDB_CODE_SUCCESS; } -- GitLab