diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e051a43f21b72aec5dbac196afb3f7407fa150fd..d69849349cb5481eb734f964bf664e5a2c6357ed 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2336,24 +2336,26 @@ typedef struct { } SVgEpSet; typedef struct { - int8_t version; // for compatibility(default 0) - int8_t intervalUnit; // MACRO: TIME_UNIT_XXX - int8_t slidingUnit; // MACRO: TIME_UNIT_XXX - int8_t timezoneInt; // sma data expired if timezone changes. - int32_t dstVgId; - char indexName[TSDB_INDEX_NAME_LEN]; - int32_t exprLen; - int32_t tagsFilterLen; - int32_t numOfVgroups; - int64_t indexUid; - tb_uid_t tableUid; // super/child/common table uid - int64_t interval; - int64_t offset; // use unit by precision of DB - int64_t sliding; - char* expr; // sma expression - char* tagsFilter; - SVgEpSet* pVgEpSet; -} STSma; // Time-range-wise SMA + int8_t version; // for compatibility(default 0) + int8_t intervalUnit; // MACRO: TIME_UNIT_XXX + int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. + int32_t dstVgId; + char indexName[TSDB_INDEX_NAME_LEN]; + int32_t exprLen; + int32_t tagsFilterLen; + int64_t indexUid; + tb_uid_t tableUid; // super/child/common table uid + tb_uid_t dstTbUid; // for dstVgroup + int64_t interval; + int64_t offset; // use unit by precision of DB + int64_t sliding; + char* dstTbName; // for dstVgroup + char* expr; // sma expression + char* tagsFilter; + SSchemaWrapper schemaRow; // for dstVgroup + SSchemaWrapper schemaTag; // for dstVgroup +} STSma; // Time-range-wise SMA typedef STSma SVCreateTSmaReq; @@ -2437,27 +2439,6 @@ static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) { return 0; } -typedef struct { - int64_t indexUid; - STimeWindow queryWindow; -} SVGetTsmaExpWndsReq; - -#define SMA_WNDS_EXPIRE_FLAG (0x1) -#define SMA_WNDS_IS_EXPIRE(flag) (((flag)&SMA_WNDS_EXPIRE_FLAG) != 0) -#define SMA_WNDS_SET_EXPIRE(flag) ((flag) |= SMA_WNDS_EXPIRE_FLAG) - -typedef struct { - int64_t indexUid; - int8_t flags; // 0x1 all window expired - int32_t numExpWnds; - TSKEY wndSKeys[]; -} SVGetTsmaExpWndsRsp; - -int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder* pCoder, const SVGetTsmaExpWndsReq* pReq); -int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder* pCoder, SVGetTsmaExpWndsReq* pReq); -int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder* pCoder, const SVGetTsmaExpWndsRsp* pReq); -int32_t tDecodeSVGetTsmaExpWndsRsp(SDecoder* pCoder, SVGetTsmaExpWndsRsp* pReq); - typedef struct { int idx; } SMCreateFullTextReq; @@ -2517,7 +2498,8 @@ typedef struct { int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp); int32_t tDeserializeSTableIndexRsp(void* buf, int32_t bufLen, STableIndexRsp* pRsp); -void tFreeSTableIndexInfo(void* pInfo); + +void tFreeSTableIndexInfo(void* pInfo); typedef struct { int8_t mqMsgType; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 3b38362647f640e0a77f07144e5128f2597b787b..7c509d6c0e638629319e462586fbc9e0c38f1300 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -190,7 +190,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) - TD_DEF_MSG_TYPE(TDMT_VND_GET_TSMA_EXP_WNDS, "vnode-get-tsma-expired-windows", SVGetTsmaExpWndsReq, SVGetTsmaExpWndsRsp) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ce6a3f2ce7280f62c56ec466ad801187c658ab04..1a53988621bf7ad6a10d1c2b4f0af4d25f7f575c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -352,9 +352,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0619) #define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061A) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061B) -#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x061C) -#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x061D) -#define TSDB_CODE_TDB_TSMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x061E) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) @@ -685,6 +682,19 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) #define TSDB_CODE_SML_INVALID_DB_CONF TAOS_DEF_ERROR_CODE(0, 0x3003) +//tsma +#define TSDB_CODE_TSMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x3100) +#define TSDB_CODE_TSMA_NO_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x3101) +#define TSDB_CODE_TSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3102) +#define TSDB_CODE_TSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3103) +#define TSDB_CODE_TSMA_NO_INDEX_IN_CACHE TAOS_DEF_ERROR_CODE(0, 0x3104) +#define TSDB_CODE_TSMA_RM_SKEY_IN_HASH TAOS_DEF_ERROR_CODE(0, 0x3105) + +//rsma +#define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150) +#define TSDB_CODE_RSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3151) + + #ifdef __cplusplus } #endif diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0abe8c6369f3fa12ab2d2863a6eb17204172690c..b7a9ef88b660dc6c2635a7cf4754806937746b32 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1634,25 +1634,31 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks break; default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - char tv[8] = {0}; - if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - GET_TYPED_DATA(v, float, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { - double v = 0; - GET_TYPED_DATA(v, double, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); + if (pCol->type == pColInfoData->info.type) { + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset, + k); } else { - uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); + char tv[8] = {0}; + if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, + k); } - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, k); } else { uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); TASSERT(0); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a7d490369639defa5f81e5d4ec141ba7211f6e39..d16ab57ea9272ea8fb748672a1b6beb94fd96e28 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3877,9 +3877,10 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1; if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1; if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1; - if (tEncodeI32(pCoder, pSma->numOfVgroups) < 0) return -1; if (tEncodeI64(pCoder, pSma->indexUid) < 0) return -1; if (tEncodeI64(pCoder, pSma->tableUid) < 0) return -1; + if (tEncodeI64(pCoder, pSma->dstTbUid) < 0) return -1; + if (tEncodeCStr(pCoder, pSma->dstTbName) < 0) return -1; if (tEncodeI64(pCoder, pSma->interval) < 0) return -1; if (tEncodeI64(pCoder, pSma->offset) < 0) return -1; if (tEncodeI64(pCoder, pSma->sliding) < 0) return -1; @@ -3889,17 +3890,10 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (pSma->tagsFilterLen > 0) { if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1; } - for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { - if (tEncodeI32(pCoder, pSma->pVgEpSet[v].vgId) < 0) return -1; - if (tEncodeI8(pCoder, pSma->pVgEpSet[v].epSet.inUse) < 0) return -1; - int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps; - if (tEncodeI8(pCoder, numOfEps) < 0) return -1; - for (int32_t n = 0; n < numOfEps; ++n) { - const SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n]; - if (tEncodeCStr(pCoder, pEp->fqdn) < 0) return -1; - if (tEncodeU16(pCoder, pEp->port) < 0) return -1; - } - } + + tEncodeSSchemaWrapper(pCoder, &pSma->schemaRow); + tEncodeSSchemaWrapper(pCoder, &pSma->schemaTag); + return 0; } @@ -3907,14 +3901,15 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { if (tDecodeI8(pCoder, &pSma->version) < 0) return -1; if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1; if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1; - if (tDecodeI32(pCoder, &pSma->dstVgId) < 0) return -1; if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1; + if (tDecodeI32(pCoder, &pSma->dstVgId) < 0) return -1; if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1; if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1; if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1; - if (tDecodeI32(pCoder, &pSma->numOfVgroups) < 0) return -1; if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1; if (tDecodeI64(pCoder, &pSma->tableUid) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->dstTbUid) < 0) return -1; + if (tDecodeCStr(pCoder, &pSma->dstTbName) < 0) return -1; if (tDecodeI64(pCoder, &pSma->interval) < 0) return -1; if (tDecodeI64(pCoder, &pSma->offset) < 0) return -1; if (tDecodeI64(pCoder, &pSma->sliding) < 0) return -1; @@ -3928,27 +3923,9 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { } else { pSma->tagsFilter = NULL; } - if (pSma->numOfVgroups > 0) { - pSma->pVgEpSet = (SVgEpSet *)tDecoderMalloc(pCoder, pSma->numOfVgroups * sizeof(SVgEpSet)); - if (!pSma->pVgEpSet) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - memset(pSma->pVgEpSet, 0, pSma->numOfVgroups * sizeof(SVgEpSet)); - - for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { - if (tDecodeI32(pCoder, &pSma->pVgEpSet[v].vgId) < 0) return -1; - if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.inUse) < 0) return -1; - if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.numOfEps) < 0) return -1; - int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps; - for (int32_t n = 0; n < numOfEps; ++n) { - SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n]; - if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1; - if (tDecodeU16(pCoder, &pEp->port) < 0) return -1; - } - } - } + // only needed in dstVgroup + tDecodeSSchemaWrapperEx(pCoder, &pSma->schemaRow); + tDecodeSSchemaWrapperEx(pCoder, &pSma->schemaTag); return 0; } @@ -3991,55 +3968,6 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { return 0; } -int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder *pCoder, const SVGetTsmaExpWndsReq *pReq) { - if (tStartEncode(pCoder) < 0) return -1; - - if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; - if (tEncodeI64(pCoder, pReq->queryWindow.skey) < 0) return -1; - if (tEncodeI64(pCoder, pReq->queryWindow.ekey) < 0) return -1; - - tEndEncode(pCoder); - return 0; -} - -int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder *pCoder, SVGetTsmaExpWndsReq *pReq) { - if (tStartDecode(pCoder) < 0) return -1; - - if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; - if (tDecodeI64(pCoder, &pReq->queryWindow.skey) < 0) return -1; - if (tDecodeI64(pCoder, &pReq->queryWindow.ekey) < 0) return -1; - - tEndDecode(pCoder); - return 0; -} - -int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder *pCoder, const SVGetTsmaExpWndsRsp *pReq) { - if (tStartEncode(pCoder) < 0) return -1; - - if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; - if (tEncodeI8(pCoder, pReq->flags) < 0) return -1; - if (tEncodeI32(pCoder, pReq->numExpWnds) < 0) return -1; - for (int32_t i = 0; i < pReq->numExpWnds; ++i) { - if (tEncodeI64(pCoder, pReq->wndSKeys[i]) < 0) return -1; - } - tEndEncode(pCoder); - return 0; -} - -int32_t tDecodeSVGetTsmaExpWndsRsp(SDecoder *pCoder, SVGetTsmaExpWndsRsp *pReq) { - if (tStartDecode(pCoder) < 0) return -1; - - if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; - if (tDecodeI8(pCoder, &pReq->flags) < 0) return -1; - if (tDecodeI32(pCoder, &pReq->numExpWnds) < 0) return -1; - for (int32_t i = 0; i < pReq->numExpWnds; ++i) { - if (tDecodeI64(pCoder, &pReq->wndSKeys[i]) < 0) return -1; - } - - tEndDecode(pCoder); - return 0; -} - int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 238ac967a2d4a90d84146af18115e0524b328e2f..4ea92a253881f42b9de4f22aa833a69ed6c8e7ce 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -298,31 +298,32 @@ typedef struct { } SVgObj; typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; - char stb[TSDB_TABLE_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createdTime; - int64_t uid; - int64_t stbUid; - int64_t dbUid; - int8_t intervalUnit; - int8_t slidingUnit; - int8_t timezone; - int32_t dstVgId; // for stream - int64_t dstTbUid; - int64_t interval; - int64_t offset; - int64_t sliding; - int32_t exprLen; // strlen + 1 - int32_t tagsFilterLen; - int32_t sqlLen; - int32_t astLen; - int32_t numOfVgroups; - char* expr; - char* tagsFilter; - char* sql; - char* ast; - SVgEpSet* pVgEpSet; + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + char dstTbName[TSDB_TABLE_FNAME_LEN]; + int64_t createdTime; + int64_t uid; + int64_t stbUid; + int64_t dbUid; + int64_t dstTbUid; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; + int32_t sqlLen; + int32_t astLen; + char* expr; + char* tagsFilter; + char* sql; + char* ast; + SSchemaWrapper schemaRow; // for dstVgroup + SSchemaWrapper schemaTag; // for dstVgroup } SSmaObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index cde36eac5802a7b10d464c4d478a3e5788dad45e..c19b558f198913a88def6c172d8427ad28f7fd2b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -26,6 +26,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "parser.h" #include "tname.h" #define TSDB_SMA_VER_NUMBER 1 @@ -82,10 +83,12 @@ static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { SDB_SET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSma->dstTbName, TSDB_DB_FNAME_LEN, _OVER) SDB_SET_INT64(pRaw, dataPos, pSma->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pSma->uid, _OVER) SDB_SET_INT64(pRaw, dataPos, pSma->stbUid, _OVER) SDB_SET_INT64(pRaw, dataPos, pSma->dbUid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->dstTbUid, _OVER) SDB_SET_INT8(pRaw, dataPos, pSma->intervalUnit, _OVER) SDB_SET_INT8(pRaw, dataPos, pSma->slidingUnit, _OVER) SDB_SET_INT8(pRaw, dataPos, pSma->timezone, _OVER) @@ -147,10 +150,12 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSma->dstTbName, TSDB_DB_FNAME_LEN, _OVER) SDB_GET_INT64(pRaw, dataPos, &pSma->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pSma->uid, _OVER) SDB_GET_INT64(pRaw, dataPos, &pSma->stbUid, _OVER) SDB_GET_INT64(pRaw, dataPos, &pSma->dbUid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->dstTbUid, _OVER) SDB_GET_INT8(pRaw, dataPos, &pSma->intervalUnit, _OVER) SDB_GET_INT8(pRaw, dataPos, &pSma->slidingUnit, _OVER) SDB_GET_INT8(pRaw, dataPos, &pSma->timezone, _OVER) @@ -260,14 +265,17 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm req.tagsFilterLen = pSma->tagsFilterLen; req.indexUid = pSma->uid; req.tableUid = pSma->stbUid; + req.dstVgId = pSma->dstVgId; + req.dstTbUid = pSma->dstTbUid; req.interval = pSma->interval; req.offset = pSma->offset; req.sliding = pSma->sliding; req.expr = pSma->expr; req.tagsFilter = pSma->tagsFilter; - req.numOfVgroups = pSma->numOfVgroups; - req.pVgEpSet = pSma->pVgEpSet; - + req.schemaRow = pSma->schemaRow; + req.schemaTag = pSma->schemaTag; + req.dstTbName = pSma->dstTbName; + // get length int32_t ret = 0; tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret); @@ -425,14 +433,30 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, mndReleaseDnode(pMnode, pDnode); // todo add sma info here - SVgEpSet *pVgEpSet = NULL; - int32_t numOfVgroups = 0; - if (mndSmaGetVgEpSet(pMnode, pDb, &pVgEpSet, &numOfVgroups) != 0) { + SNode *pAst = NULL; + if (nodesStringToNode(pSma->ast, &pAst) < 0) { + return -1; + } + if (qExtractResultSchema(pAst, &pSma->schemaRow.nCols, &pSma->schemaRow.pSchema) != 0) { + nodesDestroyNode(pAst); return -1; } + nodesDestroyNode(pAst); + pSma->schemaRow.version = 1; + + // TODO: the schemaTag generated by qExtractResultXXX later. + pSma->schemaTag.nCols = 1; + pSma->schemaTag.version = 1; + pSma->schemaTag.pSchema = taosMemoryCalloc(1, sizeof(SSchema)); + if (!pSma->schemaTag.pSchema) { + return -1; + } + pSma->schemaTag.pSchema[0].type = TSDB_DATA_TYPE_BIGINT; + pSma->schemaTag.pSchema[0].bytes = TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]; + pSma->schemaTag.pSchema[0].colId = pSma->schemaRow.nCols + PRIMARYKEY_TIMESTAMP_COL_ID; + pSma->schemaTag.pSchema[0].flags = 0; + snprintf(pSma->schemaTag.pSchema[0].name, TSDB_COL_NAME_LEN, "groupId"); - pSma->pVgEpSet = pVgEpSet; - pSma->numOfVgroups = numOfVgroups; int32_t smaContLen = 0; void *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen); @@ -464,12 +488,15 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea memcpy(smaObj.db, pDb->name, TSDB_DB_FNAME_LEN); smaObj.createdTime = taosGetTimestampMs(); smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0}; + snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "td.tsma.rst.tb.%s", pCreate->name); + memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN); + smaObj.dstTbUid = mndGenerateUid(smaObj.dstTbName, TSDB_TABLE_FNAME_LEN); smaObj.stbUid = pStb->uid; smaObj.dbUid = pStb->dbUid; smaObj.intervalUnit = pCreate->intervalUnit; smaObj.slidingUnit = pCreate->slidingUnit; smaObj.timezone = pCreate->timezone; - smaObj.dstVgId = pCreate->dstVgId; smaObj.interval = pCreate->interval; smaObj.offset = pCreate->offset; smaObj.sliding = pCreate->sliding; @@ -1087,53 +1114,4 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} - -static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups) { - SSdb *pSdb = pMnode->pSdb; - SVgObj *pVgroup = NULL; - void *pIter = NULL; - SVgEpSet *pVgEpSet = NULL; - int32_t nAllocVgs = 16; - int32_t nVgs = 0; - - pVgEpSet = taosMemoryCalloc(nAllocVgs, sizeof(SVgEpSet)); - if (!pVgEpSet) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - while (1) { - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - if (pVgroup->dbUid != pDb->uid) { - sdbRelease(pSdb, pVgroup); - continue; - } - - if (nVgs >= nAllocVgs) { - void *p = taosMemoryRealloc(pVgEpSet, nAllocVgs * 2 * sizeof(SVgEpSet)); - if (!p) { - taosMemoryFree(pVgEpSet); - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pVgroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pVgEpSet = (SVgEpSet *)p; - nAllocVgs *= 2; - } - - (pVgEpSet + nVgs)->vgId = pVgroup->vgId; - (pVgEpSet + nVgs)->epSet = mndGetVgroupEpset(pMnode, pVgroup); - - ++nVgs; - - sdbRelease(pSdb, pVgroup); - } - - *ppVgEpSet = pVgEpSet; - *numOfVgroups = nVgs; - - return 0; -} +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7abe9e3c0ddeb27906b75f50f019eec754122546..a331534a9343a66bf43b22487f60ddbae224786c 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -252,8 +252,12 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast } if (qExtractResultSchema(pAst, (int32_t *)&pStream->outputSchema.nCols, &pStream->outputSchema.pSchema) != 0) { + nodesDestroyNode(pAst); return -1; } + // free + nodesDestroyNode(pAst); + #if 0 printf("|"); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 978fd9013a24b8b14f99fe87137ac11b04cd72b1..8dca58932060ff108a1843b0c57e45f0aa76936e 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -28,7 +28,6 @@ target_sources( # sma "src/sma/sma.c" - "src/sma/smaTDBImpl.c" "src/sma/smaEnv.c" "src/sma/smaOpen.c" "src/sma/smaRollup.c" diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 4ca62f1de9fbe9183d74e9df1dfeca8fbde2e0fb..e9da1258413c6bc72350ce355391ff0c77e08063 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -43,35 +43,17 @@ typedef struct SRSmaInfo SRSmaInfo; struct SSmaEnv { TdThreadRwlock lock; int8_t type; - TXN txn; - void *pPool; // SPoolMem - SDiskID did; - TDB *dbEnv; // TODO: If it's better to put it in smaIndex level? - char *path; // relative path SSmaStat *pStat; }; #define SMA_ENV_LOCK(env) ((env)->lock) #define SMA_ENV_TYPE(env) ((env)->type) -#define SMA_ENV_DID(env) ((env)->did) -#define SMA_ENV_ENV(env) ((env)->dbEnv) -#define SMA_ENV_PATH(env) ((env)->path) #define SMA_ENV_STAT(env) ((env)->pStat) #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) struct SSmaStatItem { - /** - * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. - * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from - * Streaming Module or TSDB local persistence. - * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, - * without information about its previous state. - * - TSDB_SMA_STAT_DROPPED: 1)sma dropped - * N.B. only applicable to tsma - */ - int8_t state; // ETsdbSmaStat - SHashObj *expiredWindows; // key: skey of time window, value: version - STSma *pTSma; // cache schema + int8_t state; // ETsdbSmaStat + STSma *pTSma; // cache schema }; struct SSmaStat { @@ -84,29 +66,6 @@ struct SSmaStat { #define SMA_STAT_ITEMS(s) ((s)->smaStatItems) #define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash) -struct SSmaKey { - TSKEY skey; - int64_t groupId; -}; - -typedef struct SDBFile SDBFile; - -struct SDBFile { - int32_t fid; - TTB *pDB; - char *path; -}; - -int32_t tdSmaBeginCommit(SSmaEnv *pEnv); -int32_t tdSmaEndCommit(SSmaEnv *pEnv); - -int32_t smaOpenDBEnv(TDB **ppEnv, const char *path); -int32_t smaCloseDBEnv(TDB *pEnv); -int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF); -int32_t smaCloseDBF(SDBFile *pDBF); -int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); -void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); - void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); #if 0 @@ -114,13 +73,6 @@ int32_t tbGetTSmaStatus(SSma *pSma, STSma *param, void *result); int32_t tbRemoveTSmaData(SSma *pSma, STSma *param, STimeWindow *pWin); #endif -static FORCE_INLINE int32_t tdEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { - int32_t len = 0; - len += taosEncodeFixedI64(pData, tsKey); - len += taosEncodeFixedI64(pData, groupId); - return len; -} - int32_t tdInitSma(SSma *pSma); int32_t tdDropTSma(SSma *pSma, char *pMsg); int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid); @@ -128,13 +80,11 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg); int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat); int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); -int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); +int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType, bool onlyCheck); int32_t tdLockSma(SSma *pSma); int32_t tdUnLockSma(SSma *pSma); -int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); - static FORCE_INLINE int16_t tdTSmaAdd(SSma *pSma, int16_t n) { return atomic_add_fetch_16(&SMA_TSMA_NUM(pSma), n); } static FORCE_INLINE int16_t tdTSmaSub(SSma *pSma, int16_t n) { return atomic_sub_fetch_16(&SMA_TSMA_NUM(pSma), n); } @@ -219,11 +169,8 @@ static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDisk void *tdFreeRSmaInfo(SRSmaInfo *pInfo); int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); -int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version); -// TODO: This is the basic params, and should wrap the params to a queryHandle. -int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); - -int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); +int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9b277ede0642b73e1f66e6e97f2d7f5876b5a859..0c2b09a49343367ce9fc295579f45511869d307b 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -150,7 +150,6 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); -int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); @@ -227,7 +226,7 @@ struct SVnode { SQHandle* pQuery; }; -#define TD_VID(PVNODE) (PVNODE)->config.vgId +#define TD_VID(PVNODE) ((PVNODE)->config.vgId) #define VND_TSDB(vnd) ((vnd)->pTsdb) #define VND_RSMA0(vnd) ((vnd)->pTsdb) diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 689cd511c40cf89e465602dac341558e3e90092d..0b6a526d8cd1435a4ce3075c972549e90e69e8f4 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -34,13 +34,13 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { SMetaReader mr = {0}; // validate req + // save smaIndex metaReaderInit(&mr, pMeta, 0); if (metaGetTableEntryByUid(&mr, pCfg->indexUid) == 0) { -// TODO: just for pass case #if 1 - terrno = TSDB_CODE_TDB_TSMA_ALREADY_EXIST; + terrno = TSDB_CODE_TSMA_ALREADY_EXIST; metaReaderClear(&mr); - return -1; + return -1; // don't goto _err; #else metaReaderClear(&mr); return 0; diff --git a/source/dnode/vnode/src/sma/sma.c b/source/dnode/vnode/src/sma/sma.c index fd5dd080ca5bb6a9d81773e4add5a2a63ce9c51b..98e5d7c66d31a24ce293c0b212ba388a0210cf9b 100644 --- a/source/dnode/vnode/src/sma/sma.c +++ b/source/dnode/vnode/src/sma/sma.c @@ -36,25 +36,9 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) { return code; } -int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) { - smaWarn("vgId:%d, update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - return code; -} - -int32_t tdGetTSmaData(SSma* pSma, char* pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) { - smaWarn("vgId:%d, get tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - return code; -} - int32_t smaGetTSmaDays(SVnodeCfg* pCfg, void* pCont, uint32_t contLen, int32_t* days) { int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdGetTSmaDaysImpl(pCfg, pCont, contLen, days)) < 0) { + if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) { smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno)); } smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 179f573e8d72c3ae6938edb31e61ef6b9ec8a675..5eec5076e83a03102beb807c7a2353ccf14ed488 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -151,31 +151,11 @@ static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, return NULL; } - ASSERT(path && (strlen(path) > 0)); - SMA_ENV_PATH(pEnv) = strdup(path); - if (!SMA_ENV_PATH(pEnv)) { - tdFreeSmaEnv(pEnv); - return NULL; - } - - SMA_ENV_DID(pEnv) = did; - if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) { tdFreeSmaEnv(pEnv); return NULL; } - char aname[TSDB_FILENAME_LEN] = {0}; - tfsAbsoluteName(SMA_TFS(pSma), did, path, aname); - if (smaOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { - tdFreeSmaEnv(pEnv); - return NULL; - } - - if (!(pEnv->pPool = openPool())) { - tdFreeSmaEnv(pEnv); - return NULL; - } return pEnv; } @@ -205,10 +185,7 @@ void tdDestroySmaEnv(SSmaEnv *pSmaEnv) { if (pSmaEnv) { tdDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv)); taosMemoryFreeClear(pSmaEnv->pStat); - taosMemoryFreeClear(pSmaEnv->path); taosThreadRwlockDestroy(&(pSmaEnv->lock)); - smaCloseDBEnv(pSmaEnv->dbEnv); - closePool(pSmaEnv->pPool); } } @@ -242,7 +219,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { } /** - * 1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tdNew). + * 1. Lazy mode utilized when init SSmaStat to update expire window(or hungry mode when tdNew). * 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if * tdInitSmaStat invoked in other multithread environment later. */ @@ -280,7 +257,6 @@ void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { if (pSmaStatItem) { tDestroyTSma(pSmaStatItem->pTSma); taosMemoryFreeClear(pSmaStatItem->pTSma); - taosHashCleanup(pSmaStatItem->expiredWindows); taosMemoryFreeClear(pSmaStatItem); } return NULL; @@ -341,7 +317,7 @@ int32_t tdUnLockSma(SSma *pSma) { return 0; } -int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { +int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType, bool onlyCheck) { SSmaEnv *pEnv = NULL; // return if already init @@ -399,63 +375,3 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { return TSDB_CODE_SUCCESS; }; - -int32_t tdSmaBeginCommit(SSmaEnv *pEnv) { - TXN *pTxn = &pEnv->txn; - // start a new txn - tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - if (tdbBegin(pEnv->dbEnv, pTxn) != 0) { - smaWarn("tdSma tdb begin commit fail"); - return -1; - } - return 0; -} - -int32_t tdSmaEndCommit(SSmaEnv *pEnv) { - TXN *pTxn = &pEnv->txn; - - // Commit current txn - if (tdbCommit(pEnv->dbEnv, pTxn) != 0) { - smaWarn("tdSma tdb end commit fail"); - return -1; - } - tdbTxnClose(pTxn); - clearPool(pEnv->pPool); - return 0; -} - -#if 0 -/** - * @brief Get the start TS key of the last data block of one interval/sliding. - * - * @param pSma - * @param param - * @param result - * @return int32_t - * 1) Return 0 and fill the result if the check procedure is normal; - * 2) Return -1 if error occurs during the check procedure. - */ -int32_t tdGetTSmaStatus(SSma *pSma, void *smaIndex, void *result) { - const char *procedure = ""; - if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) { - return -1; - } - // fill the result - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Remove the tSma data files related to param between pWin. - * - * @param pSma - * @param param - * @param pWin - * @return int32_t - */ -int32_t tdRemoveTSmaData(SSma *pSma, void *smaIndex, STimeWindow *pWin) { - // for ("tSmaFiles of param-interval-sliding between pWin") { - // // remove the tSmaFile - // } - return TSDB_CODE_SUCCESS; -} -#endif diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index e738d3a4084a583852dcdf11c73d8cdafb006632..1b34529506427efd5748542e9f36b87c991e3d08 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -65,7 +65,7 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } @@ -132,7 +132,7 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui SSmaStat *pStat = SMA_ENV_STAT(pEnv); SHashObj *infoHash = NULL; if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } @@ -167,13 +167,13 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui */ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { SSma *pSma = pVnode->pSma; - SMeta *pMeta = pVnode->pMeta; - SMsgCb *pMsgCb = &pVnode->msgCb; if (!pReq->rollup) { smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); return TSDB_CODE_SUCCESS; } + SMeta *pMeta = pVnode->pMeta; + SMsgCb *pMsgCb = &pVnode->msgCb; SRSmaParam *param = &pReq->pRSmaParam; if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) { @@ -181,7 +181,7 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { return TSDB_CODE_SUCCESS; } - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP, false) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INIT_FAILED; return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/sma/smaTDBImpl.c b/source/dnode/vnode/src/sma/smaTDBImpl.c deleted file mode 100644 index cac986d053686aa33dda1a2322d22c84caddc9df..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/sma/smaTDBImpl.c +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define ALLOW_FORBID_FUNC - -#include "sma.h" - -int32_t smaOpenDBEnv(TDB **ppEnv, const char *path) { - int ret = 0; - - if (path == NULL) return -1; - - ret = tdbOpen(path, 4096, 256, ppEnv); // use as param - - if (ret != 0) { - smaError("failed to create tsdb db env, ret = %d", ret); - return -1; - } - - return 0; -} - -int32_t smaCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); } - -static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { - const SSmaKey *pKey1 = (const SSmaKey *)arg1; - const SSmaKey *pKey2 = (const SSmaKey *)arg2; - - ASSERT(len1 == len2 && len1 == sizeof(SSmaKey)); - - if (pKey1->skey < pKey2->skey) { - return -1; - } else if (pKey1->skey > pKey2->skey) { - return 1; - } - if (pKey1->groupId < pKey2->groupId) { - return -1; - } else if (pKey1->groupId > pKey2->groupId) { - return 1; - } - - return 0; -} - -static int32_t smaOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) { - tdb_cmpr_fn_t compFunc; - - // Create a database - compFunc = tdSmaKeyCmpr; - if (tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { - return -1; - } - - return 0; -} - -static int32_t smaCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); } - -int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF) { - // TEnv is shared by a group of SDBFile - if (!pEnv || !pDBF) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; - } - - // Open DBF - if (smaOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) { - smaError("failed to open DBF: %s", pDBF->path); - smaCloseDBDb(pDBF->pDB); - return -1; - } - - return 0; -} - -int32_t smaCloseDBF(SDBFile *pDBF) { - int32_t ret = 0; - if (pDBF->pDB) { - ret = smaCloseDBDb(pDBF->pDB); - pDBF->pDB = NULL; - } - taosMemoryFreeClear(pDBF->path); - return ret; -} - -int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { - int32_t ret; - - printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn); - ret = tdbTbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); - if (ret < 0) { - smaError("failed to upsert tsma data into db, ret = %d", ret); - return -1; - } - - return 0; -} - -void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) { - void *pVal = NULL; - int ret; - - ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); - - if (ret < 0) { - smaError("failed to get tsma data from db, ret = %d", ret); - return NULL; - } - - ASSERT(*valLen >= 0); - - // TODO: lock? - // TODO: Would the key/value be destoryed during return the data? - // TODO: How about the key is updated while value length is changed? The original value buffer would be freed - // automatically? - - return pVal; -} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c deleted file mode 100644 index b72be06455d8181dca8a27ea1c58cfa72ddef39f..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ /dev/null @@ -1,1037 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sma.h" -#include "tsdb.h" - -typedef STsdbCfg STSmaKeepCfg; - -#undef _TEST_SMA_PRINT_DEBUG_LOG_ -#define SMA_STORAGE_TSDB_MINUTES 86400 -#define SMA_STORAGE_TSDB_TIMES 10 -#define SMA_STORAGE_SPLIT_FACTOR 144 // least records in tsma file -#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8 -#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds - -#define SMA_STATE_ITEM_HASH_SLOT 32 - -typedef struct { - SSma *pSma; - SDBFile dFile; - const SArray *pDataBlocks; // sma data - int64_t interval; // interval with the precision of DB -} STSmaWriteH; - -typedef struct { - int32_t iter; - int32_t fid; -} SmaFsIter; - -typedef struct { - STsdb *pTsdb; - SSma *pSma; - SDBFile dFile; - int64_t interval; // interval with the precision of DB - int32_t blockSize; // size of SMA block item - int32_t days; - int8_t storageLevel; - SmaFsIter smaFsIter; -} STSmaReadH; - -typedef enum { - SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma - SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma -} ESmaStorageLevel; - -// static func - -static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); -static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval); -static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit); -static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit); -static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH); -static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel); -static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); -static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); -static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); -static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn); -// expired window - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); -static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); -static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); - -// read data - -// implementation - -/** - * @brief - * - * @param pSmaH - * @param pSma - * @param interval - * @param intervalUnit - * @return int32_t - */ -static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) { - STSmaKeepCfg *pCfg = SMA_TSDB_CFG(pSma); - pSmaH->pSma = pSma; - pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); - pSmaH->storageLevel = tdGetSmaStorageLevel(pCfg, interval); - pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Init of tSma FS - * - * @param pReadH - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { - SSma *pSma = pSmaH->pSma; - - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision)); - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); - pSmaH->dFile.path = strdup(tSmaFile); - pSmaH->smaFsIter.iter = 0; - pSmaH->smaFsIter.fid = fid; - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Set and open tSma file if it has key locates in queryWin. - * - * @param pReadH - * @param param - * @param queryWin - * @return true - * @return false - */ -static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { - // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; - // int32_t nSmaFs = taosArrayGetSize(smaFs); - - smaCloseDBF(&pReadH->dFile); - -#if 0 - while (pReadH->smaFsIter.iter < nSmaFs) { - void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); - if (pSmaFile) { // match(indexName, queryWindow) - // TODO: select the file by index_name ... - pReadH->dFile = pSmaFile; - ++pReadH->smaFsIter.iter; - break; - } - ++pReadH->smaFsIter.iter; - } - - if (pReadH->pDFile) { - tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); - return true; - } -#endif - - return false; -} - -/** - * @brief Approximate value for week/month/year. - * - * @param interval - * @param intervalUnit - * @param precision - * @param adjusted Interval already adjusted according to DB precision - * @return int64_t - */ -static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { - if (adjusted) { - return interval; - } - - switch (intervalUnit) { - case TIME_UNIT_YEAR: // approximate value - interval *= 365 * 86400 * 1e3; - break; - case TIME_UNIT_MONTH: // approximate value - interval *= 30 * 86400 * 1e3; - break; - case TIME_UNIT_WEEK: // approximate value - interval *= 7 * 86400 * 1e3; - break; - case TIME_UNIT_DAY: // the interval for tSma calculation must <= day - interval *= 86400 * 1e3; - break; - case TIME_UNIT_HOUR: - interval *= 3600 * 1e3; - break; - case TIME_UNIT_MINUTE: - interval *= 60 * 1e3; - break; - case TIME_UNIT_SECOND: - interval *= 1e3; - break; - default: - break; - } - - switch (precision) { - case TSDB_TIME_PRECISION_MILLI: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second - return interval / 1e6; - } else { // ms - return interval; - } - break; - case TSDB_TIME_PRECISION_MICRO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e3; - } else { // ms - return interval * 1e3; - } - break; - case TSDB_TIME_PRECISION_NANO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval * 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval; - } else { // ms - return interval * 1e6; - } - break; - default: // ms - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e6; - } else { // ms - return interval; - } - break; - } - return interval; -} - -static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit) { - pSmaH->pSma = pSma; - pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); - pSmaH->pDataBlocks = pDataBlocks; - pSmaH->dFile.fid = SMA_IVLD_FID; - return TSDB_CODE_SUCCESS; -} - -static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) { - if (pSmaH) { - smaCloseDBF(&pSmaH->dFile); - } -} - -static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { - SSma *pSma = pSmaH->pSma; - ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); - - pSmaH->dFile.fid = fid; - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); - pSmaH->dFile.path = strdup(tSmaFile); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pSma - * @param interval Interval calculated by DB's precision - * @param storageLevel - * @return int32_t - */ -static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) { - STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); - int32_t daysPerFile = pCfg->days; // unit is minute - - if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { - int32_t minutes = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); - if (minutes > SMA_STORAGE_TSDB_MINUTES) { - daysPerFile = SMA_STORAGE_TSDB_MINUTES; - } - } - - return daysPerFile; -} - -/** - * @brief Judge the tSma storage level - * - * @param pCfg - * @param interval - * @return int32_t - */ -static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) { - int64_t mInterval = convertTimeFromPrecisionToUnit(interval, pCfg->precision, TIME_UNIT_MINUTE); - if (pCfg->days / mInterval >= SMA_STORAGE_SPLIT_FACTOR) { - return SMA_STORAGE_LEVEL_DFILESET; - } - return SMA_STORAGE_LEVEL_TSDB; -} - -/** - * @brief Insert/Update Time-range-wise SMA data. - * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. - * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files. - * - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The - * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d). - * - The destination file of one data block for some interval is determined by its start TS key. - * - * @param pSma - * @param msg - * @return int32_t - */ -int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { - STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); - const SArray *pDataBlocks = (const SArray *)msg; - int64_t testSkey = TSKEY_INITIAL_VAL; - - // TODO: destroy SSDataBlocks(msg) - - // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus - // the sma data would arrive ahead of the update-expired-window msg. - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - if (!pDataBlocks) { - terrno = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d, insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma)); - return terrno; - } - - if (taosArrayGetSize(pDataBlocks) <= 0) { - terrno = TSDB_CODE_INVALID_PARA; - smaWarn("vgId:%d, insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } - - SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SSmaStatItem *pItem = NULL; - - tdRefSmaStat(pSma, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - - if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - - STSma *pTSma = pItem->pTSma; - STSmaWriteH tSmaH = {0}; - - if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) { - return TSDB_CODE_FAILED; - } - - char rPath[TSDB_FILENAME_LEN] = {0}; - char aPath[TSDB_FILENAME_LEN] = {0}; - snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); - tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath); - if (!taosCheckExistFile(aPath)) { - if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - } - - // Step 1: Judge the storage level and days - int32_t storageLevel = tdGetSmaStorageLevel(pCfg, tSmaH.interval); - int32_t minutePerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel); - - char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId - char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? - void *pDataBuf = NULL; - int32_t sz = taosArrayGetSize(pDataBlocks); - for (int32_t i = 0; i < sz; ++i) { - SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; - int32_t rows = pDataBlock->info.rows; - int32_t rowSize = pDataBlock->info.rowSize; - int64_t groupId = pDataBlock->info.groupId; - for (int32_t j = 0; j < rows; ++j) { - printf("|"); - TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval - void *pSmaKey = &smaKey; - bool isStartKey = false; - - int32_t tlen = 0; // reset the len - pDataBuf = &dataBuf; // reset the buf - for (int32_t k = 0; k < colNum; ++k) { - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - if (!isStartKey) { - isStartKey = true; - skey = *(TSKEY *)var; - testSkey = skey; - printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); - tdEncodeTSmaKey(groupId, skey, &pSmaKey); - } else { - printf(" %" PRIi64 " |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - } - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - printf(" %15d |", *(uint8_t *)var); - tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); - break; - case TSDB_DATA_TYPE_TINYINT: - printf(" %15d |", *(int8_t *)var); - tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf(" %15d |", *(int16_t *)var); - tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf(" %15d |", *(uint16_t *)var); - tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); - break; - case TSDB_DATA_TYPE_INT: - printf(" %15d |", *(int32_t *)var); - tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); - break; - case TSDB_DATA_TYPE_FLOAT: - printf(" %15f |", *(float *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float)); - break; - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t *)var); - tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); - break; - case TSDB_DATA_TYPE_BIGINT: - printf(" %15ld |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf(" %15lf |", *(double *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double)); - case TSDB_DATA_TYPE_UBIGINT: - printf(" %15lu |", *(uint64_t *)var); - tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); - break; - case TSDB_DATA_TYPE_NCHAR: { - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARBINARY: - // TODO: add binary/varbinary - TASSERT(0); - default: - printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - TASSERT(0); - break; - } - } - printf("\n"); - // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { - if (tlen > 0) { - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, minutePerFile, pCfg->precision)); - - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index - // file - // - Set and open the DFile or the B+Tree file - // TODO: tsdbStartTSmaCommit(); - if (fid != tSmaH.dFile.fid) { - if (tSmaH.dFile.fid != SMA_IVLD_FID) { - tdSmaEndCommit(pEnv); - smaCloseDBF(&tSmaH.dFile); - } - tdSetTSmaDataFile(&tSmaH, indexUid, fid); - smaDebug("vgId:%d, write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 - " queryKey:%" PRIi64, - SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); - if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { - smaWarn("vgId:%d, open DB file %s failed since %s", SMA_VID(pSma), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - tdSmaBeginCommit(pEnv); - } - - if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { - smaWarn("vgId:%d, insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 - " since %s", - SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno)); - tdSmaEndCommit(pEnv); - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - - smaDebug("vgId:%d, insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, - SMA_VID(pSma), indexUid, skey, groupId); - // TODO:tsdbEndTSmaCommit(); - - // Step 3: reset the SSmaStat - tdResetExpiredWindow(pSma, pStat, indexUid, skey); - } else { - smaWarn("vgId:%d, invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, - SMA_VID(pSma), skey, tlen, indexUid); - } - } - } - tdSmaEndCommit(pEnv); // TODO: not commit for every insert - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - - return TSDB_CODE_SUCCESS; -} - -int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) { - smaWarn("vgId:%d, drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - return code; -} - -/** - * @brief Insert TSma data blocks to DB File build by B+Tree - * - * @param pSmaH - * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) - * @param keyLen - * @param pData - * @param dataLen - * @return int32_t - */ -static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn) { - SDBFile *pDBFile = &pSmaH->dFile; - - // TODO: insert tsma data blocks into B+Tree(TTB) - if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { - smaWarn("vgId:%d, insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", - SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d, insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", - SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - uint32_t valueSize = 0; - void *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); - ASSERT(data != NULL); - for (uint32_t v = 0; v < valueSize; v += 8) { - smaWarn("vgId:%d, insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); - } -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief When sma data received from stream computing, make the relative expired window valid. - * - * @param pSma - * @param pStat - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { - SSmaStatItem *pItem = NULL; - - tdRefSmaStat(pSma, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { - // pItem resides in hash buffer all the time unless drop sma index - // TODO: multithread protect - if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { - // error handling - tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d, remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, - indexUid); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d, remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), - skey, indexUid); - // TODO: use a standalone interface to received state upate notification from stream computing module. - /** - * @brief state - * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. - * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. - * when batch data caculation not finised) - * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. - */ - pItem->state = TSDB_SMA_STAT_OK; - } else { - // error handling - tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d, expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid); - return TSDB_CODE_FAILED; - } - - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Drop tSma data and local cache - * - insert/query reference - * @param pSma - * @param msg - * @return int32_t - */ -static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { - SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); - - // clear local cache - if (pEnv) { - smaDebug("vgId:%d, drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid); - - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { - if (tdSmaStatIsDropped(pItem)) { - smaDebug("vgId:%d, tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - - tdWLockSmaEnv(pEnv); - if (tdSmaStatIsDropped(pItem)) { - tdUnLockSmaEnv(pEnv); - smaDebug("vgId:%d, tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - tdSmaStatSetDropped(pItem); - tdUnLockSmaEnv(pEnv); - - int32_t nSleep = 0; - int32_t refVal = INT32_MAX; - while (true) { - if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { - smaDebug("vgId:%d, drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); - break; - } - smaDebug("vgId:%d, wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); - taosSsleep(1); - if (++nSleep > SMA_DROP_EXPIRED_TIME) { - smaDebug("vgId:%d, drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); - break; - }; - } - - tdFreeSmaStatItem(pItem); - smaDebug("vgId:%d, getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid); - } - } - // clear sma data files - // TODO: - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pSma Return the data between queryWin and fill the pData. - * @param pData - * @param indexUid - * @param pQuerySKey - * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. - * @return int32_t - */ -int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); - SSmaStat *pStat = NULL; - - if (!pEnv) { - terrno = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d, getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } - - pStat = SMA_ENV_STAT(pEnv); - - tdRefSmaStat(pSma, pStat); - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { - // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if - // it's NULL. - tdUnRefSmaStat(pSma, pStat); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - smaDebug("vgId:%d, getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_FAILED; - } - -#if 0 - int32_t nQueryWin = taosArrayGetSize(pQuerySKey); - for (int32_t n = 0; n < nQueryWin; ++n) { - TSKEY skey = taosArrayGet(pQuerySKey, n); - if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - } - } -#endif - -#if 1 - int8_t smaStat = 0; - if (!tdSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query - tdUnRefSmaStat(pSma, pStat); - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - smaWarn("vgId:%d, getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, - tstrerror(terrno), smaStat); - return TSDB_CODE_FAILED; - } - - if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - smaDebug("vgId:%d, skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); - } else { - smaDebug("vgId:%d, skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); - } - - STSma *pTSma = pItem->pTSma; -#endif - -#if 1 - STSmaReadH tReadH = {0}; - tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit); - smaCloseDBF(&tReadH.dFile); - - tdUnRefSmaStat(pSma, pStat); - - tdInitTSmaFile(&tReadH, indexUid, querySKey); - smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64, - SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); - if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { - smaWarn("vgId:%d, open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - - char smaKey[SMA_KEY_LEN] = {0}; - void *pSmaKey = &smaKey; - int64_t queryGroupId = 0; - tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - - smaDebug("vgId:%d, get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, - *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); - - void *result = NULL; - int32_t valueSize = 0; - if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { - smaWarn("vgId:%d, get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", - SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); - smaCloseDBF(&tReadH.dFile); - return TSDB_CODE_FAILED; - } -#endif - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - for (uint32_t v = 0; v < valueSize; v += 8) { - smaWarn("vgId:%d, get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v)); - } -#endif - taosMemoryFreeClear(result); // TODO: fill the result to output - -#if 0 - int32_t nResult = 0; - int64_t lastKey = 0; - - while (true) { - if (nResult >= nMaxResult) { - break; - } - - // set and open the file according to the STSma param - if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) { - char bTree[100] = "\0"; - while (strncmp(bTree, "has more nodes", 100) == 0) { - if (nResult >= nMaxResult) { - break; - } - // tdGetDataFromBTree(bTree, queryWin, lastKey) - // fill the pData - ++nResult; - } - } - } -#endif - // read data from file and fill the result - smaCloseDBF(&tReadH.dFile); - return TSDB_CODE_SUCCESS; -} - -int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { - SSmaCfg *pCfg = (SSmaCfg *)pMsg; - - if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { - return -1; - } - - tdTSmaAdd(pSma, 1); - return 0; -} - -int32_t tdDropTSma(SSma *pSma, char *pMsg) { -#if 0 - SVDropTSmaReq vDropSmaReq = {0}; - if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // TODO: send msg to stream computing to drop tSma - // if ((send msg to stream computing) < 0) { - // tDestroyTSma(&vCreateSmaReq); - // return -1; - // } - // - - if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - tdTSmaSub(pSma, 1); -#endif - - // TODO: return directly or go on follow steps? - return TSDB_CODE_SUCCESS; -} - -static SSmaStatItem *tdNewSmaStatItem(int8_t state) { - SSmaStatItem *pItem = NULL; - - pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); - if (!pItem) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pItem->state = state; - pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), - true, HASH_ENTRY_LOCK); - if (!pItem->expiredWindows) { - taosMemoryFreeClear(pItem); - return NULL; - } - - return pItem; -} - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version) { - SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - if (!pItem) { - // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later - pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state - if (!pItem) { - // Response to stream computing: OOM - // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. - return TSDB_CODE_FAILED; - } - - // cache smaMeta - STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); - if (!pTSma) { - terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - smaWarn("vgId:%d, set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - pItem->pTSma = pTSma; - - if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { - // If error occurs during put smaStatItem, free the resources of pItem - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - return TSDB_CODE_FAILED; - } - } else if (!(pItem = *(SSmaStatItem **)pItem)) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) { - // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would - // tell query module to query raw TS data. - // N.B. - // 1) It is assumed to be extemely little probability event of fail to taosHashPut. - // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired - // windows failed to put into hash table. - taosHashCleanup(pItem->expiredWindows); - taosMemoryFreeClear(pItem->pTSma); - taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); - smaWarn("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, - winSKey); - return TSDB_CODE_FAILED; - } - - smaDebug("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, - winSKey); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Update expired window according to msg from stream computing module. - * - * @param pSma - * @param msg SSubmitReq - * @return int32_t - */ -int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version) { - // no time-range-sma, just return success - if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) { - smaTrace("vgId:%d, not update expire window since no tSma", SMA_VID(pSma)); - return TSDB_CODE_SUCCESS; - } - - if (!SMA_META(pSma)) { - terrno = TSDB_CODE_INVALID_PTR; - smaError("vgId:%d, update expire window failed since no meta ptr", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } - - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) { - smaError("vgId:%d, init sma env failed since %s", SMA_VID(pSma), terrstr(terrno)); - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - // Firstly, assume that tSma can only be created on super table/normal table. - // getActiveTimeWindow - - SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); - - TASSERT(pEnv && pStat && pItemsHash); - - // basic procedure - // TODO: optimization - tdRefSmaStat(pSma, pStat); - - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SInterval interval = {0}; - TSKEY lastWinSKey = INT64_MIN; - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return TSDB_CODE_FAILED; - } - - while (true) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (!pBlock) break; - - STSmaWrapper *pSW = NULL; - STSma *pTSma = NULL; - - SSubmitBlkIter blkIter = {0}; - if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - - while (true) { - STSRow *row = tGetSubmitBlkNext(&blkIter); - if (!row) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) { - if (pSW) { - pSW = tFreeTSmaWrapper(pSW, false); - } - if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) { - break; - } - if ((pSW->number) <= 0 || !pSW->tSma) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - - pTSma = pSW->tSma; - - interval.interval = pTSma->interval; - interval.intervalUnit = pTSma->intervalUnit; - interval.offset = pTSma->offset; - interval.precision = SMA_TSDB_CFG(pSma)->precision; - interval.sliding = pTSma->sliding; - interval.slidingUnit = pTSma->slidingUnit; - } - - // TODO: process multiple tsma for one table uid - TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); - - if (lastWinSKey != winSKey) { - lastWinSKey = winSKey; - if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { - pSW = tFreeTSmaWrapper(pSW, false); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - } else { - smaDebug("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", - SMA_VID(pSma), pTSma->indexUid, winSKey); - } - } - } - - tdUnRefSmaStat(pSma, pStat); - - return TSDB_CODE_SUCCESS; -} diff --git a/source/dnode/vnode/src/sma/smaTimeRange2.c b/source/dnode/vnode/src/sma/smaTimeRange2.c index 09adc1a6a269869981332a83fed3809c9206e305..9c613873abbd433db2e5539cd51adf77efd3b0f2 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange2.c +++ b/source/dnode/vnode/src/sma/smaTimeRange2.c @@ -30,54 +30,8 @@ typedef STsdbCfg STSmaKeepCfg; #define SMA_STATE_ITEM_HASH_SLOT 32 -typedef struct { - SSma *pSma; - SDBFile dFile; - const SArray *pDataBlocks; // sma data - int64_t interval; // interval with the precision of DB -} STSmaWriteH; - -typedef struct { - int32_t iter; - int32_t fid; -} SmaFsIter; - -typedef struct { - STsdb *pTsdb; - SSma *pSma; - SDBFile dFile; - int64_t interval; // interval with the precision of DB - int32_t blockSize; // size of SMA block item - int32_t days; - int8_t storageLevel; - SmaFsIter smaFsIter; -} STSmaReadH; - -typedef enum { - SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma - SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma -} ESmaStorageLevel; - // static func -static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); -static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval); -static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit); -static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit); -static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH); -static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel); -static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); -static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); -static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); -static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn); -// expired window - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); -static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); -static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); - /** * @brief Judge the tsma file split days * @@ -87,7 +41,7 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); * @param days unit is minute * @return int32_t */ -int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { +int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { SDecoder coder = {0}; tDecoderInit(&coder, pCont, contLen); @@ -130,225 +84,6 @@ _err: // implementation -/** - * @brief - * - * @param pSmaH - * @param pSma - * @param interval - * @param intervalUnit - * @return int32_t - */ -static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) { - STSmaKeepCfg *pCfg = SMA_TSDB_CFG(pSma); - pSmaH->pSma = pSma; - pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); - pSmaH->storageLevel = tdGetSmaStorageLevel(pCfg, interval); - pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Init of tSma FS - * - * @param pReadH - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { - SSma *pSma = pSmaH->pSma; - - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision)); - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); - pSmaH->dFile.path = strdup(tSmaFile); - pSmaH->smaFsIter.iter = 0; - pSmaH->smaFsIter.fid = fid; - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Set and open tSma file if it has key locates in queryWin. - * - * @param pReadH - * @param param - * @param queryWin - * @return true - * @return false - */ -static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { - // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; - // int32_t nSmaFs = taosArrayGetSize(smaFs); - - smaCloseDBF(&pReadH->dFile); - -#if 0 - while (pReadH->smaFsIter.iter < nSmaFs) { - void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); - if (pSmaFile) { // match(indexName, queryWindow) - // TODO: select the file by index_name ... - pReadH->dFile = pSmaFile; - ++pReadH->smaFsIter.iter; - break; - } - ++pReadH->smaFsIter.iter; - } - - if (pReadH->pDFile) { - tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); - return true; - } -#endif - - return false; -} - -/** - * @brief Approximate value for week/month/year. - * - * @param interval - * @param intervalUnit - * @param precision - * @param adjusted Interval already adjusted according to DB precision - * @return int64_t - */ -static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { - if (adjusted) { - return interval; - } - - switch (intervalUnit) { - case TIME_UNIT_YEAR: // approximate value - interval *= 365 * 86400 * 1e3; - break; - case TIME_UNIT_MONTH: // approximate value - interval *= 30 * 86400 * 1e3; - break; - case TIME_UNIT_WEEK: // approximate value - interval *= 7 * 86400 * 1e3; - break; - case TIME_UNIT_DAY: // the interval for tSma calculation must <= day - interval *= 86400 * 1e3; - break; - case TIME_UNIT_HOUR: - interval *= 3600 * 1e3; - break; - case TIME_UNIT_MINUTE: - interval *= 60 * 1e3; - break; - case TIME_UNIT_SECOND: - interval *= 1e3; - break; - default: - break; - } - - switch (precision) { - case TSDB_TIME_PRECISION_MILLI: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second - return interval / 1e6; - } else { // ms - return interval; - } - break; - case TSDB_TIME_PRECISION_MICRO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e3; - } else { // ms - return interval * 1e3; - } - break; - case TSDB_TIME_PRECISION_NANO: - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval * 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval; - } else { // ms - return interval * 1e6; - } - break; - default: // ms - if (TIME_UNIT_MICROSECOND == intervalUnit) { // us - return interval / 1e3; - } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns - return interval / 1e6; - } else { // ms - return interval; - } - break; - } - return interval; -} - -static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit) { - pSmaH->pSma = pSma; - pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); - pSmaH->pDataBlocks = pDataBlocks; - pSmaH->dFile.fid = SMA_IVLD_FID; - return TSDB_CODE_SUCCESS; -} - -static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) { - if (pSmaH) { - smaCloseDBF(&pSmaH->dFile); - } -} - -static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { - SSma *pSma = pSmaH->pSma; - ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); - - pSmaH->dFile.fid = fid; - char tSmaFile[TSDB_FILENAME_LEN] = {0}; - snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); - pSmaH->dFile.path = strdup(tSmaFile); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pSma - * @param interval Interval calculated by DB's precision - * @param storageLevel - * @return int32_t - */ -static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) { - STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); - int32_t daysPerFile = pCfg->days; // unit is minute - - if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { - int32_t minutes = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); - if (minutes > SMA_STORAGE_TSDB_MINUTES) { - daysPerFile = SMA_STORAGE_TSDB_MINUTES; - } - } - - return daysPerFile; -} - -/** - * @brief Judge the tSma storage level - * - * @param pCfg - * @param interval - * @return int32_t - */ -static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) { - int64_t mInterval = convertTimeFromPrecisionToUnit(interval, pCfg->precision, TIME_UNIT_MINUTE); - if (pCfg->days / mInterval >= SMA_STORAGE_SPLIT_FACTOR) { - return SMA_STORAGE_LEVEL_DFILESET; - } - return SMA_STORAGE_LEVEL_TSDB; -} - /** * @brief Insert/Update Time-range-wise SMA data. * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. @@ -362,28 +97,28 @@ static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) { * @return int32_t */ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { - STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + const SArray *pDataBlocks = (const SArray *)msg; - int64_t testSkey = TSKEY_INITIAL_VAL; // TODO: destroy SSDataBlocks(msg) // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus // the sma data would arrive ahead of the update-expired-window msg. - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE, false) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INIT_FAILED; return TSDB_CODE_FAILED; } if (!pDataBlocks) { terrno = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma)); return terrno; } if (taosArrayGetSize(pDataBlocks) <= 0) { terrno = TSDB_CODE_INVALID_PARA; - smaWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma)); + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is empty", SMA_VID(pSma)); return TSDB_CODE_FAILED; } @@ -398,463 +133,15 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { } if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) { - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + terrno = TSDB_CODE_TSMA_INVALID_STAT; tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; } - STSma *pTSma = pItem->pTSma; - STSmaWriteH tSmaH = {0}; - - if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) { - return TSDB_CODE_FAILED; - } - - char rPath[TSDB_FILENAME_LEN] = {0}; - char aPath[TSDB_FILENAME_LEN] = {0}; - snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); - tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath); - if (!taosCheckExistFile(aPath)) { - if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - } - - // Step 1: Judge the storage level and days - int32_t storageLevel = tdGetSmaStorageLevel(pCfg, tSmaH.interval); - int32_t minutePerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel); - - char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId - char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? - void *pDataBuf = NULL; - int32_t sz = taosArrayGetSize(pDataBlocks); - for (int32_t i = 0; i < sz; ++i) { - SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; - int32_t rows = pDataBlock->info.rows; - int32_t rowSize = pDataBlock->info.rowSize; - int64_t groupId = pDataBlock->info.groupId; - for (int32_t j = 0; j < rows; ++j) { - printf("|"); - TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval - void *pSmaKey = &smaKey; - bool isStartKey = false; - - int32_t tlen = 0; // reset the len - pDataBuf = &dataBuf; // reset the buf - for (int32_t k = 0; k < colNum; ++k) { - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - if (!isStartKey) { - isStartKey = true; - skey = *(TSKEY *)var; - testSkey = skey; - printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); - tdEncodeTSmaKey(groupId, skey, &pSmaKey); - } else { - printf(" %" PRIi64 " |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - } - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - printf(" %15d |", *(uint8_t *)var); - tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); - break; - case TSDB_DATA_TYPE_TINYINT: - printf(" %15d |", *(int8_t *)var); - tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf(" %15d |", *(int16_t *)var); - tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf(" %15d |", *(uint16_t *)var); - tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); - break; - case TSDB_DATA_TYPE_INT: - printf(" %15d |", *(int32_t *)var); - tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); - break; - case TSDB_DATA_TYPE_FLOAT: - printf(" %15f |", *(float *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float)); - break; - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t *)var); - tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); - break; - case TSDB_DATA_TYPE_BIGINT: - printf(" %15ld |", *(int64_t *)var); - tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf(" %15lf |", *(double *)var); - tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double)); - case TSDB_DATA_TYPE_UBIGINT: - printf(" %15lu |", *(uint64_t *)var); - tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); - break; - case TSDB_DATA_TYPE_NCHAR: { - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - char tmpChar[100] = {0}; - strncpy(tmpChar, varDataVal(var), varDataLen(var)); - printf(" %s |", tmpChar); - tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); - break; - } - case TSDB_DATA_TYPE_VARBINARY: - // TODO: add binary/varbinary - TASSERT(0); - default: - printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - TASSERT(0); - break; - } - } - printf("\n"); - // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { - if (tlen > 0) { - int32_t fid = (int32_t)(TSDB_KEY_FID(skey, minutePerFile, pCfg->precision)); - - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index - // file - // - Set and open the DFile or the B+Tree file - // TODO: tsdbStartTSmaCommit(); - if (fid != tSmaH.dFile.fid) { - if (tSmaH.dFile.fid != SMA_IVLD_FID) { - tdSmaEndCommit(pEnv); - smaCloseDBF(&tSmaH.dFile); - } - tdSetTSmaDataFile(&tSmaH, indexUid, fid); - smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 - " queryKey:%" PRIi64, - SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); - if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { - smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - tdSmaBeginCommit(pEnv); - } - - if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { - smaWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 - " since %s", - SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno)); - tdSmaEndCommit(pEnv); - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - - smaDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, - SMA_VID(pSma), indexUid, skey, groupId); - // TODO:tsdbEndTSmaCommit(); - - // Step 3: reset the SSmaStat - tdResetExpiredWindow(pSma, pStat, indexUid, skey); - } else { - smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, - SMA_VID(pSma), skey, tlen, indexUid); - } - } - } - tdSmaEndCommit(pEnv); // TODO: not commit for every insert - tdDestroyTSmaWriteH(&tSmaH); - tdUnRefSmaStat(pSma, pStat); - - return TSDB_CODE_SUCCESS; -} - -int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) { - smaWarn("vgId:%d drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - return code; -} - -/** - * @brief Insert TSma data blocks to DB File build by B+Tree - * - * @param pSmaH - * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) - * @param keyLen - * @param pData - * @param dataLen - * @return int32_t - */ -static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn) { - SDBFile *pDBFile = &pSmaH->dFile; - - // TODO: insert tsma data blocks into B+Tree(TTB) - if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { - smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", - SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", - SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - uint32_t valueSize = 0; - void *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); - ASSERT(data != NULL); - for (uint32_t v = 0; v < valueSize; v += 8) { - smaWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); - } -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief When sma data received from stream computing, make the relative expired window valid. - * - * @param pSma - * @param pStat - * @param indexUid - * @param skey - * @return int32_t - */ -static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { - SSmaStatItem *pItem = NULL; - - tdRefSmaStat(pSma, pStat); - - if (pStat && SMA_STAT_ITEMS(pStat)) { - pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); - } - if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { - // pItem resides in hash buffer all the time unless drop sma index - // TODO: multithread protect - if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { - // error handling - tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, - indexUid); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), - skey, indexUid); - // TODO: use a standalone interface to received state upate notification from stream computing module. - /** - * @brief state - * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. - * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. - * when batch data caculation not finised) - * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. - */ - pItem->state = TSDB_SMA_STAT_OK; - } else { - // error handling - tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid); - return TSDB_CODE_FAILED; - } - - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Drop tSma data and local cache - * - insert/query reference - * @param pSma - * @param msg - * @return int32_t - */ -static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { - SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); - - // clear local cache - if (pEnv) { - smaDebug("vgId:%d drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid); - - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { - if (tdSmaStatIsDropped(pItem)) { - smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - - tdWLockSmaEnv(pEnv); - if (tdSmaStatIsDropped(pItem)) { - tdUnLockSmaEnv(pEnv); - smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode - } - tdSmaStatSetDropped(pItem); - tdUnLockSmaEnv(pEnv); - - int32_t nSleep = 0; - int32_t refVal = INT32_MAX; - while (true) { - if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { - smaDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); - break; - } - smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); - taosSsleep(1); - if (++nSleep > SMA_DROP_EXPIRED_TIME) { - smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); - break; - }; - } - - tdFreeSmaStatItem(pItem); - smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid); - } - } - // clear sma data files - // TODO: - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pSma Return the data between queryWin and fill the pData. - * @param pData - * @param indexUid - * @param pQuerySKey - * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. - * @return int32_t - */ -int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { - SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); - SSmaStat *pStat = NULL; - - if (!pEnv) { - terrno = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } - - pStat = SMA_ENV_STAT(pEnv); - - tdRefSmaStat(pSma, pStat); - SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { - // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if - // it's NULL. - tdUnRefSmaStat(pSma, pStat); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid); - return TSDB_CODE_FAILED; - } - -#if 0 - int32_t nQueryWin = taosArrayGetSize(pQuerySKey); - for (int32_t n = 0; n < nQueryWin; ++n) { - TSKEY skey = taosArrayGet(pQuerySKey, n); - if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - } - } -#endif - -#if 1 - int8_t smaStat = 0; - if (!tdSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query - tdUnRefSmaStat(pSma, pStat); - terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, - tstrerror(terrno), smaStat); - return TSDB_CODE_FAILED; - } - - if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { - // TODO: mark this window as expired. - smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); - } else { - smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); - } - STSma *pTSma = pItem->pTSma; -#endif - -#if 1 - STSmaReadH tReadH = {0}; - tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit); - smaCloseDBF(&tReadH.dFile); tdUnRefSmaStat(pSma, pStat); - tdInitTSmaFile(&tReadH, indexUid, querySKey); - smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64, - SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); - if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { - smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - - char smaKey[SMA_KEY_LEN] = {0}; - void *pSmaKey = &smaKey; - int64_t queryGroupId = 0; - tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - - smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, - *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); - - void *result = NULL; - int32_t valueSize = 0; - if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { - smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", - SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); - smaCloseDBF(&tReadH.dFile); - return TSDB_CODE_FAILED; - } -#endif - -#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ - for (uint32_t v = 0; v < valueSize; v += 8) { - smaWarn("vgId:%d get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v)); - } -#endif - taosMemoryFreeClear(result); // TODO: fill the result to output - -#if 0 - int32_t nResult = 0; - int64_t lastKey = 0; - - while (true) { - if (nResult >= nMaxResult) { - break; - } - - // set and open the file according to the STSma param - if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) { - char bTree[100] = "\0"; - while (strncmp(bTree, "has more nodes", 100) == 0) { - if (nResult >= nMaxResult) { - break; - } - // tdGetDataFromBTree(bTree, queryWin, lastKey) - // fill the pData - ++nResult; - } - } - } -#endif - // read data from file and fill the result - smaCloseDBF(&tReadH.dFile); return TSDB_CODE_SUCCESS; } @@ -865,224 +152,19 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { return -1; } - tdTSmaAdd(pSma, 1); - return 0; -} - -int32_t tdDropTSma(SSma *pSma, char *pMsg) { -#if 0 - SVDropTSmaReq vDropSmaReq = {0}; - if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // TODO: send msg to stream computing to drop tSma - // if ((send msg to stream computing) < 0) { - // tDestroyTSma(&vCreateSmaReq); - // return -1; - // } - // + if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { + // create stable to save tsma result in dstVgId + SVCreateStbReq pReq = {0}; + pReq.name = pCfg->dstTbName; + pReq.suid = pCfg->dstTbUid; + pReq.schemaRow = pCfg->schemaRow; + pReq.schemaTag = pCfg->schemaTag; - if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) { - // TODO: handle error - return -1; - } - - tdTSmaSub(pSma, 1); -#endif - - // TODO: return directly or go on follow steps? - return TSDB_CODE_SUCCESS; -} - -static SSmaStatItem *tdNewSmaStatItem(int8_t state) { - SSmaStatItem *pItem = NULL; - - pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); - if (!pItem) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pItem->state = state; - pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), - true, HASH_ENTRY_LOCK); - if (!pItem->expiredWindows) { - taosMemoryFreeClear(pItem); - return NULL; - } - - return pItem; -} - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version) { - SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - if (!pItem) { - // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later - pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state - if (!pItem) { - // Response to stream computing: OOM - // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. - return TSDB_CODE_FAILED; - } - - // cache smaMeta - STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); - if (!pTSma) { - terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - smaWarn("vgId:%d set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - pItem->pTSma = pTSma; - - if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { - // If error occurs during put smaStatItem, free the resources of pItem - taosHashCleanup(pItem->expiredWindows); - taosMemoryFree(pItem); - return TSDB_CODE_FAILED; - } - } else if (!(pItem = *(SSmaStatItem **)pItem)) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) { - // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would - // tell query module to query raw TS data. - // N.B. - // 1) It is assumed to be extemely little probability event of fail to taosHashPut. - // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired - // windows failed to put into hash table. - taosHashCleanup(pItem->expiredWindows); - taosMemoryFreeClear(pItem->pTSma); - taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); - smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, - winSKey); - return TSDB_CODE_FAILED; - } - - smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, - winSKey); - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Update expired window according to msg from stream computing module. - * - * @param pSma - * @param msg SSubmitReq - * @return int32_t - */ -int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version) { - // no time-range-sma, just return success - if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) { - smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma)); - return TSDB_CODE_SUCCESS; - } - - if (!SMA_META(pSma)) { - terrno = TSDB_CODE_INVALID_PTR; - smaError("vgId:%d update expire window failed since no meta ptr", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } - - if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) { - smaError("vgId:%d init sma env failed since %s", SMA_VID(pSma), terrstr(terrno)); - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - // Firstly, assume that tSma can only be created on super table/normal table. - // getActiveTimeWindow - - SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); - - TASSERT(pEnv && pStat && pItemsHash); - - // basic procedure - // TODO: optimization - tdRefSmaStat(pSma, pStat); - - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SInterval interval = {0}; - TSKEY lastWinSKey = INT64_MIN; - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return TSDB_CODE_FAILED; - } - - while (true) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (!pBlock) break; - - STSmaWrapper *pSW = NULL; - STSma *pTSma = NULL; - - SSubmitBlkIter blkIter = {0}; - if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - - while (true) { - STSRow *row = tGetSubmitBlkNext(&blkIter); - if (!row) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) { - if (pSW) { - pSW = tFreeTSmaWrapper(pSW, false); - } - if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) { - break; - } - if ((pSW->number) <= 0 || !pSW->tSma) { - pSW = tFreeTSmaWrapper(pSW, false); - break; - } - - pTSma = pSW->tSma; - - interval.interval = pTSma->interval; - interval.intervalUnit = pTSma->intervalUnit; - interval.offset = pTSma->offset; - interval.precision = SMA_TSDB_CFG(pSma)->precision; - interval.sliding = pTSma->sliding; - interval.slidingUnit = pTSma->slidingUnit; - } - - // TODO: process multiple tsma for one table uid - TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); - - if (lastWinSKey != winSKey) { - lastWinSKey = winSKey; - if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { - pSW = tFreeTSmaWrapper(pSW, false); - tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_FAILED; - } - } else { - smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", - SMA_VID(pSma), pTSma->indexUid, winSKey); - } + if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { + return -1; } } - tdUnRefSmaStat(pSma, pStat); - - return TSDB_CODE_SUCCESS; + tdTSmaAdd(pSma, 1); + return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 9be94eb5b6f9f0c45230b31f507e7f55ee551bd7..3af8901b2b20f97f66ea78209ddb3d9936fc93c1 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -238,9 +238,6 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (msgType == TDMT_VND_SUBMIT) { if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0; - if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) { - // TODO handle sma error - } void* data = taosMemoryMalloc(msgLen); if (data == NULL) { return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 17b8afda4b46f3d909705e9e7574bf4c304485d0..90093f2510ea976c8e55f96f6a24d8bb43de0fc5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -150,7 +150,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { return code; _err: - tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index ffbef4e765fd073034cce7c58005a64e2e5a54fd..d8bc87d47142d59d3564c3cbcf89bfe4d6a0e84e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -176,13 +176,13 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid pMemTable->nDelOp++; - tsdbError("vgId:%d delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 + tsdbError("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " since %s", TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); return code; _err: - tsdbError("vgId:%d failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 + tsdbError("vgId:%d, failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " since %s", TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); return code; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 51ed73969300c818085c6f28839ccf5182ba782e..086694bdf593ee9db4c22c2dcf1bbd6478f2fcbf 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -225,6 +225,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("message in fetch queue is processing"); char *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + switch (pMsg->msgType) { case TDMT_VND_FETCH: return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg, 0); @@ -236,13 +237,10 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg, 0); case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg, 0); - case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg); - case TDMT_VND_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId); - case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH: @@ -279,7 +277,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRp void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO - // blockDebugShowData(data); + // blockDebugShowData(data, __func__); tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 3b8c94e413ee866441f5d7514e13986d31fc4137..0161fac9b5aeb6b389b86655ac98ed6168fd41d6 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -373,7 +373,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks); EXPECT_NE(pTsdb->pTfs, nullptr); - // generate SSubmitReq msg and update expired window + // generate SSubmitReq msg and update expire window int16_t schemaVer = 0; uint32_t mockRowLen = sizeof(STSRow); uint32_t mockRowNum = 2; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b5c54b780dc90182be3f266b7d9502b3de617e02..e122ad0ab6ef10afe82a65d874db34131d332b04 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -353,9 +353,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag valu TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST, "TSMA already exists") // query @@ -537,25 +534,38 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DELETE_WHERE, "The DELETE statemen TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG, "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes") //planner -TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "Planner internal error") +TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "Planner internal error") //udf -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR, "udf pipe read error") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR, "udf pipe connect error") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE, "udf no function handle") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE, "udf invalid bufsize") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output type") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR, "udf pipe read error") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR, "udf pipe connect error") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE, "udf no function handle") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE, "udf invalid bufsize") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output type") //schemaless -TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") -TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") -TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data type") -TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF, "Invalid schemaless db config") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF, "Invalid schemaless db config") + +//tsma +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_ALREADY_EXIST, "Tsma already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_META, "No tsma index in meta") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_ENV, "Invalid tsma env") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_STAT, "Invalid tsma state") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE, "No tsma index in cache") +TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_RM_SKEY_IN_HASH, "Rm tsma skey in cache") + +//rsma +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT, "Invalid rsma state") + #ifdef TAOS_ERROR_C };