diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 71be1a501406243bc50fad2cd32f34e6af78f349..bc2046b940229a93984316836e11fbba1c8c8460 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2255,20 +2255,22 @@ static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { } } -static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) { +static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { if (pSW) { if (pSW->tSma) { - for (uint32_t i = 0; i < pSW->number; ++i) { - tdDestroyTSma(pSW->tSma + i); + if (deepCopy) { + for (uint32_t i = 0; i < pSW->number; ++i) { + tdDestroyTSma(pSW->tSma + i); + } } taosMemoryFreeClear(pSW->tSma); } } } -static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) { - tdDestroyTSmaWrapper(pSW); - taosMemoryFree(pSW); +static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { + tdDestroyTSmaWrapper(pSW, deepCopy); + taosMemoryFreeClear(pSW); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 869bf0c0a90676da27d1362340cd5a2b6eb192ea..a9963651fd2bcf923848da1233873c7deb73ca56 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -530,6 +530,7 @@ static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq) { pStream = mndAcquireStream(pMnode, createReq.name); if (pStream != NULL) { mError("sma:%s, failed to create since stream:%s already exist", createReq.name, createReq.name); + terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; goto _OVER; } @@ -565,7 +566,7 @@ static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mError("sma:%s, failed to create since %s", createReq.name, terrstr()); + mError("sma:%s, failed to create since %s", createReq.name, terrstr(terrno)); } mndReleaseStb(pMnode, pStb); diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index c5ca806829c0abcb72c1511019abd99c338e9e39..d3abc95da9c7cc141ff99f42e7517957444c2497 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -24,7 +24,6 @@ extern "C" { typedef struct SMetaIdx SMetaIdx; typedef struct SMetaDB SMetaDB; -typedef struct SMSmaCursor SMSmaCursor; // metaDebug ================== // clang-format off @@ -114,22 +113,12 @@ typedef struct { int64_t smaUid; } SSmaIdxKey; -#if 1 - -SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); -void metaCloseSmaCursor(SMSmaCursor* pSmaCur); -int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); - #ifndef META_REFACT // SMetaDB int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); -int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); -int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); -#endif - #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 76a30f58ddde9e87226b21837719304d6c9e7f80..a58e1808a6211341ab0b49a71b3d3edca5581f94 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -218,6 +218,11 @@ static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDisk void *tdFreeRSmaInfo(SRSmaInfo *pInfo); +int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); +int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version); +// TODO: This is the basic params, and should wrap the params to a queryHandle. +int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 2018f0a68ca956937c2cf1ab30034c616dd38624..c25abf650923f53bb067fe88ad83d6909e3dbc35 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -89,11 +89,13 @@ STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaGetTbNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); -void metaCloseCtbCurosr(SMCtbCursor* pCtbCur); +void metaCloseCtbCursor(SMCtbCursor* pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); -SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup); -void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode); -STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid); +STSma* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid); +STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy); +SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid); +SArray* metaGetSmaTbUids(SMeta* pMeta); + int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -126,7 +128,7 @@ int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version); -int32_t tdProcessTSmaCreate(SSma* pSma, char* pMsg); +int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); int32_t tdProcessRSmaCreate(SSma* pSma, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 2bc0d7517db42084ef5aeb22785b189427ae8a8c..84a895777191e5af05cc8519c2584f5dbee5600b 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -69,6 +69,11 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1; if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; } else if (pME->type == TSDB_TSMA_TABLE) { + pME->smaEntry.tsma = taosMemoryCalloc(1, sizeof(STSma)); + if(!pME->smaEntry.tsma) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } if (tDecodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; } else { ASSERT(0); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 1e2c94679f3983c7f5c654027d4bd1f554fd369a..369f16b43088cc763bd1e4c1dee14b829c355a96 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -225,7 +225,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { return pCtbCur; } -void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { +void metaCloseCtbCursor(SMCtbCursor *pCtbCur) { if (pCtbCur) { if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta); if (pCtbCur->pCur) { @@ -291,178 +291,268 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { return pTSchema; } -STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { -#if 0 -#ifdef META_TDB_SMA_TEST - STSmaWrapper *pSW = NULL; +int metaGetTbNum(SMeta *pMeta) { + // TODO + // ASSERT(0); + return 0; +} - pSW = taosMemoryCalloc(1, sizeof(*pSW)); - if (pSW == NULL) { +typedef struct { + SMeta *pMeta; + TDBC *pCur; + tb_uid_t uid; + void *pKey; + void *pVal; + int kLen; + int vLen; +} SMSmaCursor; + +SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { + SMSmaCursor *pSmaCur = NULL; + SSmaIdxKey smaIdxKey; + int ret; + int c; + + pSmaCur = (SMSmaCursor *)taosMemoryCalloc(1, sizeof(*pSmaCur)); + if (pSmaCur == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); - if (pCur == NULL) { - taosMemoryFree(pSW); + pSmaCur->pMeta = pMeta; + pSmaCur->uid = uid; + metaRLock(pMeta); + + ret = tdbDbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL); + if (ret < 0) { + metaULock(pMeta); + taosMemoryFree(pSmaCur); return NULL; } - void *pBuf = NULL; - SSmaIdxKey *pSmaIdxKey = NULL; + // move to the suid + smaIdxKey.uid = uid; + smaIdxKey.smaUid = INT64_MIN; + tdbDbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c); + if (c > 0) { + tdbDbcMoveToNext(pSmaCur->pCur); + } - while (true) { - // TODO: lock during iterate? - if (tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { - pSmaIdxKey = pCur->pKey; - ASSERT(pSmaIdxKey != NULL); + return pSmaCur; +} - void *pSmaVal = metaGetSmaInfoByIndex(pMeta, pSmaIdxKey->smaUid, false); +void metaCloseSmaCursor(SMSmaCursor *pSmaCur) { + if (pSmaCur) { + if (pSmaCur->pMeta) metaULock(pSmaCur->pMeta); + if (pSmaCur->pCur) { + tdbDbcClose(pSmaCur->pCur); - if (pSmaVal == NULL) { - tsdbWarn("no tsma exists for indexUid: %" PRIi64, pSmaIdxKey->smaUid); - continue; - } + tdbFree(pSmaCur->pKey); + tdbFree(pSmaCur->pVal); + } - ++pSW->number; - STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); - if (tptr == NULL) { - tdbFree(pSmaVal); - metaCloseSmaCursor(pCur); - tdDestroyTSmaWrapper(pSW); - taosMemoryFreeClear(pSW); - return NULL; + taosMemoryFree(pSmaCur); + } +} + +tb_uid_t metaSmaCursorNext(SMSmaCursor *pSmaCur) { + int ret; + SSmaIdxKey *pSmaIdxKey; + + ret = tdbDbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen); + if (ret < 0) { + return 0; + } + + pSmaIdxKey = pSmaCur->pKey; + if (pSmaIdxKey->uid > pSmaCur->uid) { + return 0; + } + + return pSmaIdxKey->uid; +} + +STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { + STSmaWrapper *pSW = NULL; + SArray *pSmaIds = NULL; + + if (!(pSmaIds = metaGetSmaIdsByTable(pMeta, uid))) { + return NULL; + } + + pSW = taosMemoryCalloc(1, sizeof(*pSW)); + if (!pSW) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + pSW->number = taosArrayGetSize(pSmaIds); + pSW->tSma = taosMemoryCalloc(pSW->number, sizeof(STSma)); + + if (!pSW->tSma) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); + if (pCur == NULL) { + goto _err; + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, pMeta, 0); + int64_t smaId; + int smaIdx = 0; + STSma *pTSma = NULL; + for (int i = 0; i < pSW->number; ++i) { + smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i); + if (metaGetTableEntryByUid(&mr, smaId) < 0) { + metaWarn("vgId:%d no entry for tbId: %" PRIi64 ", smaId: %" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); + continue; + } + pTSma = pSW->tSma + smaIdx; + memcpy(pTSma, mr.me.smaEntry.tsma, sizeof(STSma)); + if (deepCopy) { + if (pTSma->exprLen > 0) { + if (!(pTSma->expr = taosMemoryCalloc(1, pTSma->exprLen))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + memcpy((void*)pTSma->expr, mr.me.smaEntry.tsma->expr, pTSma->exprLen); } - pSW->tSma = tptr; - pBuf = pSmaVal; - if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { - tdbFree(pSmaVal); - metaCloseSmaCursor(pCur); - tdDestroyTSmaWrapper(pSW); - taosMemoryFreeClear(pSW); - return NULL; + if (pTSma->tagsFilterLen > 0) { + if (!(pTSma->tagsFilter = taosMemoryCalloc(1, pTSma->tagsFilterLen))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } - tdbFree(pSmaVal); - continue; + memcpy((void*)pTSma->tagsFilter, mr.me.smaEntry.tsma->tagsFilter, pTSma->tagsFilterLen); + } else { + pTSma->exprLen = 0; + pTSma->expr = NULL; + pTSma->tagsFilterLen = 0; + pTSma->tagsFilter = NULL; } - break; + + ++smaIdx; } - metaCloseSmaCursor(pCur); + if (smaIdx <= 0) goto _err; + pSW->number = smaIdx; + metaReaderClear(&mr); + taosArrayDestroy(pSmaIds); + metaCloseSmaCursor(pCur); return pSW; - -#endif -#endif +_err: + metaReaderClear(&mr); + taosArrayDestroy(pSmaIds); + metaCloseSmaCursor(pCur); + tdFreeTSmaWrapper(pSW, deepCopy); return NULL; } -int metaGetTbNum(SMeta *pMeta) { - // TODO - // ASSERT(0); - return 0; +STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { + STSma *pTSma = NULL; + SMetaReader mr = {0}; + metaReaderInit(&mr, pMeta, 0); + if (metaGetTableEntryByUid(&mr, indexUid) < 0) { + metaWarn("vgId:%d failed to get table entry for smaId: %" PRIi64, TD_VID(pMeta->pVnode), indexUid); + metaReaderClear(&mr); + return NULL; + } + pTSma = (STSma *)taosMemoryMalloc(sizeof(STSma)); + if (!pTSma) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + metaReaderClear(&mr); + return NULL; + } + + memcpy(pTSma, mr.me.smaEntry.tsma, sizeof(STSma)); + + metaReaderClear(&mr); + return pTSma; } -SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { -#if 0 - // TODO - // ASSERT(0); // comment this line to pass CI - // return NULL: -#ifdef META_TDB_SMA_TEST - SArray *pUids = NULL; - SMetaDB *pDB = pMeta->pDB; - void *pKey; +SArray *metaGetSmaIdsByTable(SMeta *pMeta, tb_uid_t uid) { + SArray *pUids = NULL; + SSmaIdxKey *pSmaIdxKey = NULL; - // TODO: lock? - SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0); - if (pCur == NULL) { + SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); + if (!pCur) { return NULL; } - // TODO: lock? - SSmaIdxKey *pSmaIdxKey = NULL; - tb_uid_t uid = 0; - while (true) { - // TODO: lock during iterate? - if (tdbDbcNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { - ASSERT(pSmaIdxKey != NULL); - pSmaIdxKey = pCur->pKey; - - if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) { - continue; - } - uid = pSmaIdxKey->uid; + while (1) { + tb_uid_t id = metaSmaCursorNext(pCur); + if (id == 0) { + break; + } + if (!pUids) { + pUids = taosArrayInit(16, sizeof(tb_uid_t)); if (!pUids) { - pUids = taosArrayInit(16, sizeof(tb_uid_t)); - if (!pUids) { - metaCloseSmaCursor(pCur); - return NULL; - } + terrno = TSDB_CODE_OUT_OF_MEMORY; + metaCloseSmaCursor(pCur); + return NULL; } + } - taosArrayPush(pUids, &uid); + pSmaIdxKey = (SSmaIdxKey *)pCur->pKey; - continue; + if (taosArrayPush(pUids, &pSmaIdxKey->smaUid) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + metaCloseSmaCursor(pCur); + taosArrayDestroy(pUids); + return NULL; } - break; } metaCloseSmaCursor(pCur); - return pUids; -#endif -#endif - return NULL; } -void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { -#if 0 - // TODO - // ASSERT(0); - // return NULL; -#ifdef META_TDB_SMA_TEST - SMetaDB *pDB = pMeta->pDB; - void *pKey = NULL; - void *pVal = NULL; - int kLen = 0; - int vLen = 0; - int ret = -1; - - // Set key - pKey = (void *)&indexUid; - kLen = sizeof(indexUid); - - // Query - ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen); - if (ret != 0 || !pVal) { +SArray *metaGetSmaTbUids(SMeta *pMeta) { + SArray *pUids = NULL; + SSmaIdxKey *pSmaIdxKey = NULL; + tb_uid_t lastUid = 0; + + SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0); + if (!pCur) { return NULL; } - if (!isDecode) { - // return raw value - return pVal; - } + while (1) { + tb_uid_t uid = metaSmaCursorNext(pCur); + if (uid == 0) { + break; + } - // Decode - STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma)); - if (pCfg == NULL) { - taosMemoryFree(pVal); - return NULL; - } + if (lastUid == uid) { + continue; + } - void *pBuf = pVal; - if (tDecodeTSma(pBuf, pCfg) == NULL) { - tdDestroyTSma(pCfg); - taosMemoryFree(pCfg); - tdbFree(pVal); - return NULL; + lastUid = uid; + + if (!pUids) { + pUids = taosArrayInit(16, sizeof(tb_uid_t)); + if (!pUids) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + metaCloseSmaCursor(pCur); + return NULL; + } + } + + if (taosArrayPush(pUids, &uid) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + metaCloseSmaCursor(pCur); + taosArrayDestroy(pUids); + return NULL; + } } - tdbFree(pVal); - return pCfg; -#endif -#endif - return NULL; + metaCloseSmaCursor(pCur); + return pUids; } #endif diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 8ce7ea589514089111ddac993dd21729fed9ff4f..d0a35411520847c347d92ef135db011f6bc7930d 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -16,7 +16,7 @@ #include "meta.h" static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME); -static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME); +static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME); int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { // TODO: Validate the cfg @@ -81,55 +81,6 @@ int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { return TSDB_CODE_SUCCESS; } -// static int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { -// int32_t ret = 0; -// void *pBuf = NULL, *qBuf = NULL; -// void *key = {0}, *val = {0}; - -// // save sma info -// int32_t len = tEncodeTSma(NULL, pSmaCfg); -// pBuf = taosMemoryCalloc(1, len); -// if (pBuf == NULL) { -// terrno = TSDB_CODE_OUT_OF_MEMORY; -// return -1; -// } - -// key = (void *)&pSmaCfg->indexUid; -// qBuf = pBuf; -// tEncodeTSma(&qBuf, pSmaCfg); -// val = pBuf; - -// int32_t kLen = sizeof(pSmaCfg->indexUid); -// int32_t vLen = POINTER_DISTANCE(qBuf, pBuf); - -// ret = tdbDbInsert(pMeta->pTbDb, key, kLen, val, vLen, &pMeta->txn); -// if (ret < 0) { -// taosMemoryFreeClear(pBuf); -// return -1; -// } - -// // add sma idx -// SSmaIdxKey smaIdxKey; -// smaIdxKey.uid = pSmaCfg->tableUid; -// smaIdxKey.smaUid = pSmaCfg->indexUid; -// key = &smaIdxKey; -// kLen = sizeof(smaIdxKey); -// val = NULL; -// vLen = 0; - -// ret = tdbDbInsert(pMeta->pSmaIdx, key, kLen, val, vLen, &pMeta->txn); -// if (ret < 0) { -// taosMemoryFreeClear(pBuf); -// return -1; -// } - -// // release -// taosMemoryFreeClear(pBuf); - -// return 0; -// } - - static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { STbDbKey tbDbKey; void *pKey = NULL; @@ -182,6 +133,10 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } +static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { + return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); +} + static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; @@ -194,9 +149,13 @@ static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { // save to table.db if (metaSaveSmaToDB(pMeta, pME) < 0) goto _err; - // // update uid.idx + // update uid.idx if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err; + // update name.idx + if (metaUpdateNameIdx(pMeta, pME) < 0) goto _err; + + // update sma.idx if (metaUpdateSmaIdx(pMeta, pME) < 0) goto _err; metaULock(pMeta); diff --git a/source/dnode/vnode/src/sma/sma.c b/source/dnode/vnode/src/sma/sma.c index 2c54e10087ae3893dd131749d49db6a05ceb4234..0e7ce385a1c2aa225d21201af2fcc7f0ffd72d79 100644 --- a/source/dnode/vnode/src/sma/sma.c +++ b/source/dnode/vnode/src/sma/sma.c @@ -27,4 +27,28 @@ int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) { return code; } +int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) { + int32_t code = TSDB_CODE_SUCCESS; + + if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) { + smaWarn("vgId:%d create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + // TODO: destroy SSDataBlocks(msg) + return code; +} +int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) { + smaWarn("vgId:%d update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} + +int32_t tdGetTSmaData(SSma* pSma, char* pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) { + smaWarn("vgId:%d get tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index c02276f5fe6aecb12389d66c9d6b870545c9c19b..8285b74e509f53a8ed3a9d2e5745d2f56135087e 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -122,12 +122,10 @@ static void poolFree(void *arg, void *ptr) { } int32_t tdInitSma(SSma *pSma) { - // tSma - int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(SMA_META(pSma), false)); + int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(SMA_META(pSma))); if (numOfTSma > 0) { atomic_store_16(&SMA_TSMA_NUM(pSma), (int16_t)numOfTSma); } - // TODO: rSma return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index f9cb5a1a09e5a8e20a6f8b8e21e23a99c1980d49..88af049d0bd298e58e51286e0980fd13a7872734 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -443,7 +443,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { // TODO: use the proper schema instead of 0, and cache STSchema in cache - STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, 0); + STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, 1); if (!pTSchema) { terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index b04885c5f02a4d44c3b5750459f1edd5c2ee43db..357cf710a25227c1363374cee0688ff5baa71252 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -70,15 +70,15 @@ static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, TXN *txn); // expired window -static int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version); + + static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); // read data -// TODO: This is the basic params, and should wrap the params to a queryHandle. -static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); + // implementation @@ -713,7 +713,7 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. * @return int32_t */ -static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { +int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); SSmaStat *pStat = NULL; @@ -834,35 +834,15 @@ static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKE return TSDB_CODE_SUCCESS; } -int32_t tdProcessTSmaCreate(SSma *pSma, char *pMsg) { - #if 0 - SSmaCfg vCreateSmaReq = {0}; - if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - smaWarn("vgId:%d tsma create msg received but deserialize failed since %s", SMA_VID(pSma), terrstr(terrno)); - return -1; - } - - smaDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", SMA_VID(pSma), - vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid); +int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { + SSmaCfg *pCfg = (SSmaCfg *)pMsg; - // record current timezone of server side - vCreateSmaReq.tSma.timezoneInt = tsTimezone; - - if (metaCreateTSma(SMA_META(pSma), &vCreateSmaReq) < 0) { - // TODO: handle error - smaWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", SMA_VID(pSma), - vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno)); - tdDestroyTSma(&vCreateSmaReq.tSma); + if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { return -1; } tdTSmaAdd(pSma, 1); - - tdDestroyTSma(&vCreateSmaReq.tSma); - // TODO: return directly or go on follow steps? -#endif - return TSDB_CODE_SUCCESS; + return 0; } int32_t tdDropTSma(SSma *pSma, char *pMsg) { @@ -930,7 +910,7 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde } // cache smaMeta - STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid, true); + STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); if (!pTSma) { terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; taosHashCleanup(pItem->expiredWindows); @@ -1031,25 +1011,25 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) SSubmitBlkIter blkIter = {0}; if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { - pSW = tdFreeTSmaWrapper(pSW); + pSW = tdFreeTSmaWrapper(pSW, false); break; } while (true) { STSRow *row = tGetSubmitBlkNext(&blkIter); if (!row) { - tdFreeTSmaWrapper(pSW); + pSW = tdFreeTSmaWrapper(pSW, false); break; } - if (!pSW || (pTSma->tableUid != pBlock->suid)) { + if (!pSW || (pTSma->tableUid != msgIter.suid)) { if (pSW) { - pSW = tdFreeTSmaWrapper(pSW); + pSW = tdFreeTSmaWrapper(pSW, false); } - if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), pBlock->suid))) { + if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) { break; } if ((pSW->number) <= 0 || !pSW->tSma) { - pSW = tdFreeTSmaWrapper(pSW); + pSW = tdFreeTSmaWrapper(pSW, false); break; } @@ -1068,6 +1048,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) if (lastWinSKey != winSKey) { lastWinSKey = winSKey; if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { + pSW = tdFreeTSmaWrapper(pSW, false); tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; } @@ -1083,21 +1064,3 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) return TSDB_CODE_SUCCESS; } - -int32_t tdUpdateExpireWindow(SSma *pSma, SSubmitReq *pMsg, int64_t version) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) { - smaWarn("vgId:%d update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - return code; -} - -int32_t tdGetTSmaData(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) { - smaWarn("vgId:%d get tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - return code; -} - - diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 55fe8a3945260caf44ec0f5aab55fd213989d3cd..c1be8db473e8735e70c38c15df720fc0f35855a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2760,7 +2760,7 @@ static int32_t getAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { taosArrayPush(list, &info); } - metaCloseCtbCurosr(pCur); + metaCloseCtbCursor(pCur); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 43d68fc9fdf90f58fa895909e50e385491d90ff7..9991afc4aedb02b44e477f5766a56c4005fabf1f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -711,15 +711,22 @@ static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq goto _err; } - if (metaCreateTSma(pVnode->pMeta, version, &req) < 0) { + // record current timezone of server side + req.timezoneInt = tsTimezone; + + if (tdProcessTSmaCreate(pVnode->pSma, version, (const char *)&req) < 0) { pRsp->code = terrno; goto _err; } tDecoderClear(&coder); + vDebug("vgId:%d success to create tsma %s:%" PRIi64 " for table %" PRIi64, TD_VID(pVnode), req.indexName, + req.indexUid, req.tableUid); return 0; _err: tDecoderClear(&coder); + vError("vgId:%d failed to create tsma %s:%" PRIi64 " for table %" PRIi64 " since %s", TD_VID(pVnode), req.indexName, + req.indexUid, req.tableUid, terrstr(terrno)); return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b5e64242e4e50179352debf1989a97f70108eef5..e78bf16f52e4b1c77b45dc232c69935d518fb3df 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -358,7 +358,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST, "Tsma already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST, "TSMA already exists") // query diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index f6bc9f83064bca6da216fae4e23260e8bff83645..18fe3b9afe9776b1d9d0e15767f307265e229ef6 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -104,7 +104,8 @@ ./test.sh -f tsim/mnode/basic1.sim -m # --- sma -# ./test.sh -f tsim/sma/tsmaCreateInsertData.sim +./test.sh -f tsim/sma/tsmaCreateInsertData.sim +./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim # --- valgrind ./test.sh -f tsim/valgrind/checkError.sim -v diff --git a/tests/script/tsim/sma/rsmaCreateInsertQuery.sim b/tests/script/tsim/sma/rsmaCreateInsertQuery.sim new file mode 100644 index 0000000000000000000000000000000000000000..f6de5cc2aec9b927a01ad3d04bf81cefafa87493 --- /dev/null +++ b/tests/script/tsim/sma/rsmaCreateInsertQuery.sim @@ -0,0 +1,89 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database with retentions +sql create database d0 retentions 15s:7d,1m:21d,15m:365d; +sql use d0 + +print =============== create super table and register rsma +sql create table if not exists stb (ts timestamp, c1 int) tags (city binary(20),district binary(20)) rollup(min) file_factor 0.1 delay 2; + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags("BeiJing", "ChaoYang"); + +sql show tables +if $rows != 1 then + return -1 +endi + +print =============== insert data and trigger rollup +sql insert into ct1 values(now, 10); +sql insert into ct1 values(now+1s, 1); +sql insert into ct1 values(now+2s, 100); + + +print =============== select * from retention level 2 from memory +sql select * from ct1; +print $data00 $data01 +if $rows > 1 then + print retention level 2 file rows $rows > 1 + return -1 +endi +print =============== select * from retention level 1 from memory +sql select * from ct1 where ts > now-8d; +print $data00 $data01 +if $rows > 1 then + print retention level 1 file rows $rows > 1 + return -1 +endi +print =============== select * from retention level 0 from memory +sql select * from ct1 where ts > now-3d; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +if $rows < 1 then + print retention level 0 file rows $rows < 1 + return -1 +endi +#=================================================================== + + +#==================== reboot to trigger commit data to file +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +print =============== select * from retention level 2 from file +sql select * from ct1; +print $data00 $data01 +if $rows > 1 then + print retention level 2 file rows $rows > 1 + return -1 +endi + +print =============== select * from retention level 1 from file +sql select * from ct1 where ts > now-8d; +print $data00 $data01 +if $rows > 1 then + print retention level 1 file rows $rows > 1 + return -1 +endi + +print =============== select * from retention level 0 from file +sql select * from ct1 where ts > now-3d; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +if $rows < 1 then + print retention level 0 file rows $rows < 1 + return -1 +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file