diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a05cc0224a1a4c9d82579721cd7870ec81766686..326f00dbbd5c4565c03b3c8b253d511d42b7668c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,7 +12,6 @@ ADD_SUBDIRECTORY(client) ADD_SUBDIRECTORY(query) ADD_SUBDIRECTORY(kit) ADD_SUBDIRECTORY(plugins) -ADD_SUBDIRECTORY(sdb) ADD_SUBDIRECTORY(mnode) ADD_SUBDIRECTORY(vnode) ADD_SUBDIRECTORY(dnode) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index e9d231171512c8cb9c54b0ababf386bfa821dbf2..6bec41d146ee0ad5d8cf313f9e697768310e97af 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -32,7 +32,7 @@ extern "C" { #define UTIL_TABLE_IS_SUPERTABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE)) #define UTIL_TABLE_IS_NOMRAL_TABLE(metaInfo) (!(UTIL_TABLE_IS_SUPERTABLE(metaInfo))) -#define UTIL_TABLE_CREATE_FROM_STABLE(metaInfo) \ +#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE)) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) @@ -106,7 +106,7 @@ bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryOnMetric(SSqlCmd* pCmd); -bool tscQueryMetricTags(SQueryInfo* pQueryInfo); +bool tscQueryTags(SQueryInfo* pQueryInfo); bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd); void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, @@ -176,8 +176,8 @@ void tscIncStreamExecutionCount(void* pStream); bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); // get starter position of metric query condition (query on tags) in SSqlCmd.payload -SCond* tsGetMetricQueryCondPos(STagCond* pCond, uint64_t tableIndex); -void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str); +SCond* tsGetSTableQueryCondPos(STagCond* pCond, uint64_t tableIndex); +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str); void tscTagCondCopy(STagCond* dest, const STagCond* src); void tscTagCondRelease(STagCond* pCond); @@ -207,7 +207,7 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd); void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* keyStr, uint64_t uid); -int tscGetMetricMeta(SSqlObj* pSql, int32_t clauseIndex); +int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 2dd580dda00651d8d7d4d84505147a8f464d954c..9cb43a9f3666f330704d968528986b38246b7351 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -70,8 +70,9 @@ typedef struct STableMeta { typedef struct STableMetaInfo { STableMeta * pTableMeta; // table meta, cached in client side and acquried by name - SSuperTableMeta *pMetricMeta; // metricmeta - +// SSuperTableMeta *pMetricMeta; // metricmeta + SArray* vgroupIdList; + /* * 1. keep the vnode index during the multi-vnode super table projection query * 2. keep the vnode index for multi-vnode insertion @@ -294,20 +295,21 @@ typedef struct SResRec { struct STSBuf; typedef struct { - int32_t code; int64_t numOfRows; // num of results in current retrieved int64_t numOfTotal; // num of total results int64_t numOfTotalInCurrentClause; // num of total result in current subclause char * pRsp; - int rspType; - int rspLen; + int32_t rspType; + int32_t rspLen; uint64_t qhandle; int64_t uid; int64_t useconds; int64_t offset; // offset value from vnode during projection query of stable - int row; + int32_t row; int16_t numOfCols; int16_t precision; + bool completed; + int32_t code; int32_t numOfGroups; SResRec * pGroupRec; char * data; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 2bab6e03fe0ab4fd746d4c246299252b4f85fe44..985a6741f95ded4e3f430e5ee656434d2f4cef42 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -406,11 +406,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pSql->fp == NULL) { - tscError("%p callBack is NULL!!!", pSql); - return; - } - if (pSql->fp == (void *)1) { pSql->fp = NULL; @@ -464,7 +459,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - code = tscGetMetricMeta(pSql, 0); + code = tscGetSTableVgroupInfo(pSql, 0); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; @@ -494,7 +489,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - code = tscGetMetricMeta(pSql, pCmd->clauseIndex); + code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; @@ -519,7 +514,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscTansformSQLFunctionForSTableQuery(pQueryInfo); tscIncStreamExecutionCount(pSql->pStream); } else { - tscTrace("%p get tableMeta/metricMeta successfully", pSql); + tscTrace("%p get tableMeta successfully", pSql); } tscDoQuery(pSql); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 689f9715d08b4569817d16daca224c2482c7643c..c7be1b84cefa4271f8dae73c9dd3235b6597f0e0 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -417,6 +417,10 @@ static void count_function(SQLFunctionCtx *pCtx) { numOfElem += 1; } } else { + /* + * when counting on the primary time stamp column and no statistics data is provided, + * simple use the size value + */ numOfElem = pCtx->size; } } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 46f3ab6687d8fed3110460a5e66f30d0843e8297..42b04d7a16ed53bec33f395fbe2d05adae6541a6 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -286,6 +286,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { // todo add order support static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { +#if 0 // the result structure has been completed in sql parse, so we // only need to reorganize the results in the column format SSqlCmd * pCmd = &pSql->cmd; @@ -337,6 +338,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { } } +#endif return 0; } @@ -345,7 +347,7 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - +#if 0 SSuperTableMeta *pMetricMeta = tscGetMetaInfo(pQueryInfo, 0)->pMetricMeta; int32_t totalNumOfResults = 1; // count function only produce one result int32_t rowLen = tscGetResRowLength(pQueryInfo); @@ -369,7 +371,8 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { } rowIdx++; } - +#endif + return 0; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 8fb6b925efd01fc1e22be509895762868fe28bc6..6d01538d8fef438eb18fe05f5adf6bd313cbd7f3 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -18,9 +18,10 @@ #define _XOPEN_SOURCE -#include "hash.h" #include "os.h" -#include "tscSecondaryMerge.h" + +#include "hash.h" +//#include "tscSecondaryMerge.h" #include "tscUtil.h" #include "tschemautil.h" #include "tsclient.h" @@ -32,6 +33,8 @@ #include "tstoken.h" #include "ttime.h" +#include "dataformat.h" + enum { TSDB_USE_SERVER_TS = 0, TSDB_USE_CLI_TS = 1, @@ -393,7 +396,6 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error, int16_t timePrec, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - // bool isPrevOptr; //fang, never used SSQLToken sToken = {0}; char * payload = pDataBlocks->pData + pDataBlocks->size; @@ -604,8 +606,8 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 return TSDB_CODE_SUCCESS; } -static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { - pBlocks->sid = pTableMeta->sid; +static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { + pBlocks->tid = pTableMeta->sid; pBlocks->uid = pTableMeta->uid; pBlocks->sversion = pTableMeta->sversion; pBlocks->numOfRows += numOfRows; @@ -613,10 +615,10 @@ static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const STableMeta *pTableM // data block is disordered, sort it in ascending order void sortRemoveDuplicates(STableDataBlocks *dataBuf) { - SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)dataBuf->pData; + SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; // size is less than the total size, since duplicated rows may be removed yet. - assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SShellSubmitBlock) == dataBuf->size); + assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size); // if use server time, this block must be ordered if (dataBuf->tsSource == TSDB_USE_SERVER_TS) { @@ -624,7 +626,7 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) { } if (!dataBuf->ordered) { - char *pBlockData = pBlocks->payLoad; + char *pBlockData = pBlocks->data; qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); int32_t i = 0; @@ -650,7 +652,7 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) { dataBuf->ordered = true; pBlocks->numOfRows = i + 1; - dataBuf->size = sizeof(SShellSubmitBlock) + dataBuf->rowSize * pBlocks->numOfRows; + dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; } } @@ -663,7 +665,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char STableDataBlocks *dataBuf = NULL; int32_t ret = tscGetDataBlockFromList(pTableHashList, pCmd->pDataBlocks, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SShellSubmitBlock), tinfo.rowSize, pTableMetaInfo->name, + sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &dataBuf); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -691,11 +693,11 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char SParamInfo *param = dataBuf->params + i; if (param->idx == -1) { param->idx = pCmd->numOfParams++; - param->offset -= sizeof(SShellSubmitBlock); + param->offset -= sizeof(SSubmitBlk); } } - SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(dataBuf->pData); + SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); dataBuf->vgId = pTableMeta->vgId; @@ -1141,7 +1143,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { STableDataBlocks *pDataBlock = NULL; STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - int32_t ret = tscCreateDataBlock(PATH_MAX, tinfo.rowSize, sizeof(SShellSubmitBlock), pTableMetaInfo->name, + int32_t ret = tscCreateDataBlock(PATH_MAX, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pDataBlock); if (ret != TSDB_CODE_SUCCESS) { goto _error_clean; @@ -1236,22 +1238,10 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _clean; } - // submit to more than one vnode - if (pCmd->pDataBlocks->nSize > 0) { - // merge according to vgId + if (pCmd->pDataBlocks->nSize > 0) { // merge according to vgId if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { goto _error_clean; } - - STableDataBlocks *pDataBlock = pCmd->pDataBlocks->pData[0]; - if ((code = tscCopyDataBlockToPayload(pSql, pDataBlock)) != TSDB_CODE_SUCCESS) { - goto _error_clean; - } - - pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - - // set the next sent data vnode index in data block arraylist - pTableMetaInfo->vnodeIndex = 1; } else { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); } @@ -1303,12 +1293,7 @@ int tsParseInsertSql(SSqlObj *pSql) { int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion) { int32_t ret = TSDB_CODE_SUCCESS; - - if (NULL == pSql->asyncTblPos) { - tscCleanSqlCmd(&pSql->cmd); - } else { - tscTrace("continue parse sql: %s", pSql->asyncTblPos); - } + tscTrace("continue parse sql: %s", pSql->asyncTblPos); if (tscIsInsertOrImportData(pSql->sqlstr)) { /* @@ -1353,7 +1338,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock assert(pCmd->numOfClause == 1); STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta; - SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(pTableDataBlocks->pData); + SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData); tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { @@ -1394,7 +1379,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { pCmd->pDataBlocks = tscCreateBlockArrayList(); STableDataBlocks *pTableDataBlock = NULL; - int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, rowSize, sizeof(SShellSubmitBlock), + int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pTableDataBlock); if (ret != TSDB_CODE_SUCCESS) { return -1; @@ -1435,7 +1420,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { } pTableDataBlock = pCmd->pDataBlocks->pData[0]; - pTableDataBlock->size = sizeof(SShellSubmitBlock); + pTableDataBlock->size = sizeof(SSubmitBlk); pTableDataBlock->rowSize = tinfo.rowSize; numOfRows += pSql->res.numOfRows; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 96215ce73c269c1a2ec0d99bb0d7a6a217b97396..241f24a74777be21ffb557209f20d227ff4ef957 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -325,12 +325,12 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) { STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i]; - uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock); + uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); uint32_t dataSize = totalDataSize / alloced; assert(dataSize * alloced == totalDataSize); if (alloced == binded) { - totalDataSize += dataSize + sizeof(SShellSubmitBlock); + totalDataSize += dataSize + sizeof(SSubmitBlk); if (totalDataSize > pBlock->nAllocSize) { const double factor = 1.5; void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); @@ -342,7 +342,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { } } - char* data = pBlock->pData + sizeof(SShellSubmitBlock) + dataSize * binded; + char* data = pBlock->pData + sizeof(SSubmitBlk) + dataSize * binded; for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { SParamInfo* param = pBlock->params + j; int code = doBindParam(data, param, bind + param->idx); @@ -365,10 +365,10 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) { STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i]; - uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock); + uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); pBlock->size += totalDataSize / alloced; - SShellSubmitBlock* pSubmit = (SShellSubmitBlock*)pBlock->pData; + SSubmitBlk* pSubmit = (SSubmitBlk*)pBlock->pData; pSubmit->numOfRows += pSubmit->numOfRows / alloced; } @@ -398,10 +398,10 @@ static int insertStmtReset(STscStmt* pStmt) { for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) { STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i]; - uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock); - pBlock->size = sizeof(SShellSubmitBlock) + totalDataSize / alloced; + uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); + pBlock->size = sizeof(SSubmitBlk) + totalDataSize / alloced; - SShellSubmitBlock* pSubmit = (SShellSubmitBlock*)pBlock->pData; + SSubmitBlk* pSubmit = (SSubmitBlk*)pBlock->pData; pSubmit->numOfRows = pSubmit->numOfRows / alloced; } } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e0850a7139f72a7569560c8131d5c8c2ff69fa14..2505d544e481a4ad98d212ba4908dae29add35b6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -209,9 +209,14 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); - assert(pQueryInfo->numOfTables == 0); +// assert(pQueryInfo->numOfTables == 0); - STableMetaInfo* pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); + STableMetaInfo* pTableMetaInfo = NULL; + if (pQueryInfo->numOfTables == 0) { + pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); + } else { + pTableMetaInfo = &pQueryInfo->pTableMetaInfo[0]; + } pCmd->command = pInfo->type; @@ -639,7 +644,7 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { * check invalid SQL: * select tbname, tags_fields from super_table_name interval(1s) */ - if (tscQueryMetricTags(pQueryInfo) && pQueryInfo->intervalTime > 0) { + if (tscQueryTags(pQueryInfo) && pQueryInfo->intervalTime > 0) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -746,7 +751,7 @@ int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlO tscClearMeterMetaInfo(pTableMetaInfo, false); } } else { - assert(pTableMetaInfo->pTableMeta == NULL && pTableMetaInfo->pMetricMeta == NULL); + assert(pTableMetaInfo->pTableMeta == NULL); } tfree(oldName); @@ -1252,7 +1257,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (tscQueryMetricTags(pQueryInfo)) { // local handle the metric tag query + if (tscQueryTags(pQueryInfo)) { // local handle the metric tag query pCmd->count = numOfCols; // the number of meter schema, tricky. pQueryInfo->command = TSDB_SQL_RETRIEVE_TAGS; } @@ -1293,7 +1298,7 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c int16_t functionId = (int16_t)((colIdx >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); if (functionId == TSDB_FUNC_TAGPRJ) { - addRequiredTagColumn(pQueryInfo, colIdx - numOfCols, tableIndex); +// addRequiredTagColumn(pQueryInfo, colIdx - numOfCols, tableIndex); pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; } else { pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; @@ -1396,8 +1401,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum SColumnList ids = {0}; ids.ids[0] = *pIndex; - // tag columns do not add to source list - ids.num = (j >= tscGetNumOfColumns(pTableMeta)) ? 0 : 1; + ids.num = 1; insertResultField(pQueryInfo, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name, pExpr); } @@ -1489,11 +1493,9 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, resColIdx, functionID, pColIndex, type, bytes, bytes); strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName)); - // for point interpolation/last_row query, we need the timestamp column to be loaded + // for all querie, the timestamp column meeds to be loaded SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { - tscColumnBaseInfoInsert(pQueryInfo, &index); - } + tscColumnBaseInfoInsert(pQueryInfo, &index); SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); insertResultField(pQueryInfo, resColIdx, &ids, bytes, type, columnName, pExpr); @@ -1581,7 +1583,10 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); } } - + + SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscColumnBaseInfoInsert(pQueryInfo, &tsCol); + return TSDB_CODE_SUCCESS; } case TK_SUM: @@ -1689,7 +1694,10 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); } } - + + SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscColumnBaseInfoInsert(pQueryInfo, &tsCol); + return TSDB_CODE_SUCCESS; } case TK_FIRST: @@ -1708,7 +1716,6 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt } /* in first/last function, multiple columns can be add to resultset */ - for (int32_t i = 0; i < pItem->pNode->pParam->nExpr; ++i) { tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[i]); if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { @@ -1753,7 +1760,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt } } } - + return TSDB_CODE_SUCCESS; } else { // select * from xxx int32_t numOfFields = 0; @@ -1773,6 +1780,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); } + return TSDB_CODE_SUCCESS; } } @@ -1891,6 +1899,8 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt default: return TSDB_CODE_INVALID_SQL; } + + } // todo refactor @@ -2779,7 +2789,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { +static int32_t tablenameListToString(tSQLExpr* pExpr, SStringBuilder* sb) { tSQLExprList* pList = pExpr->pParam; if (pList->nExpr <= 0) { return TSDB_CODE_INVALID_SQL; @@ -2805,7 +2815,7 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuild return TSDB_CODE_SUCCESS; } -static int32_t tablenameCondToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { +static int32_t tablenameCondToString(tSQLExpr* pExpr, SStringBuilder* sb) { taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); taosStringBuilderAppendString(sb, pExpr->val.pz); @@ -3501,7 +3511,7 @@ int tableNameCompar(const void* lhs, const void* rhs) { return ret > 0 ? 1 : -1; } -static int32_t setTableCondForMetricQuery(SQueryInfo* pQueryInfo, const char* account, tSQLExpr* pExpr, +static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* account, tSQLExpr* pExpr, int16_t tableCondIndex, SStringBuilder* sb) { const char* msg = "table name too long"; @@ -3730,7 +3740,7 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, return ret; } - tsSetMetricQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c); + tsSetSTableQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c); doCompactQueryExpr(pExpr); tSQLExprDestroy(p1); @@ -3746,8 +3756,8 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql return TSDB_CODE_SUCCESS; } - const char* msg = "invalid filter expression"; const char* msg1 = "invalid expression"; + const char* msg2 = "invalid filter expression"; int32_t ret = TSDB_CODE_SUCCESS; @@ -3805,11 +3815,11 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql // 7. query condition for table name pQueryInfo->tagCond.relType = (condExpr.relType == TK_AND) ? TSDB_RELATION_AND : TSDB_RELATION_OR; - ret = setTableCondForMetricQuery(pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb); + ret = setTableCondForSTableQuery(pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb); taosStringBuilderDestroy(&sb); if (!validateFilterExpr(pQueryInfo)) { - return invalidSqlErrMsg(pQueryInfo->msg, msg); + return invalidSqlErrMsg(pQueryInfo->msg, msg2); } doAddJoinTagsColumnsIntoTagList(pQueryInfo, &condExpr); @@ -4234,7 +4244,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo))) { return invalidSqlErrMsg(pQueryInfo->msg, msg4); } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) && - UTIL_TABLE_CREATE_FROM_STABLE(pTableMetaInfo)) { + UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg6); } @@ -4660,14 +4670,13 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* * And then launching multiple async-queries against all qualified virtual nodes, during the first-stage * query operation. */ - int32_t code = tscGetMetricMeta(pSql, clauseIndex); + int32_t code = tscGetSTableVgroupInfo(pSql, clauseIndex); if (code != TSDB_CODE_SUCCESS) { return code; } // No tables included. No results generated. Query results are empty. - SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; - if (pTableMetaInfo->pTableMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfTables == 0) { + if (pTableMetaInfo->pTableMeta == NULL) { tscTrace("%p no table in metricmeta, no output result", pSql); pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; } @@ -5435,8 +5444,9 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); // two table: the first one is for current table, and the secondary is for the super table. - tscAddEmptyMetaInfo(pQueryInfo); - assert(pQueryInfo->numOfTables == 2); + if (pQueryInfo->numOfTables < 2) { + tscAddEmptyMetaInfo(pQueryInfo); + } const int32_t TABLE_INDEX = 0; const int32_t STABLE_INDEX = 1; @@ -5681,6 +5691,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr); + + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + int32_t code = tscGetSTableVgroupInfo(pSql, index); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } // parse the group by clause in the first place if (parseGroupbyClause(pQueryInfo, pQuerySql->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index be0065ff4a0773c009513beb5bddefe80b67c3e1..2492352284afd2a571d011a4dffa21ef10b932e5 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -32,7 +32,7 @@ int32_t tscGetNumOfTags(const STableMeta* pTableMeta) { } if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) { - assert(tinfo.numOfTags > 0); + assert(tinfo.numOfTags >= 0); return tinfo.numOfTags; } @@ -51,14 +51,14 @@ int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) { SSchema *tscGetTableSchema(const STableMeta *pTableMeta) { assert(pTableMeta != NULL); - if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - STableMeta* pSTableMeta = pTableMeta->pSTable; - assert (pSTableMeta != NULL); - - return pSTableMeta->schema; - } +// if (pTableMeta->tableType == TSDB_CHILD_TABLE) { +// STableMeta* pSTableMeta = pTableMeta->pSTable; +// assert (pSTableMeta != NULL); +// +// return pSTableMeta->schema; +// } - return pTableMeta->schema; + return (SSchema*) pTableMeta->schema; } SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) { @@ -72,12 +72,14 @@ SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) { STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); - + +#if 0 if (pTableMeta->tableType == TSDB_CHILD_TABLE) { assert (pTableMeta->pSTable != NULL); return pTableMeta->pSTable->tableInfo; } - +#endif + return pTableMeta->tableInfo; } @@ -130,12 +132,13 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t startCol) assert(pTableMeta != NULL); SSchema* pSchema = pTableMeta->schema; - +#if 0 if (pTableMeta->tableType == TSDB_CHILD_TABLE) { assert (pTableMeta->pSTable != NULL); pSchema = pTableMeta->pSTable->schema; } - +#endif + return &pSchema[startCol]; } diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 0ee8afd53c59969f11bfed85c1c32804e48050f9..1d4ed51f32a68cb39d511f83c5ea1f47a3aeb4eb 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -605,7 +605,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pTableMetaInfo->pMetricMeta->numOfVnodes); + (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * 1); if (*pMemBuffer == NULL) { tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -636,7 +636,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); - for (int32_t i = 0; i < pTableMetaInfo->pMetricMeta->numOfVnodes; ++i) { + size_t numOfSubs = taosArrayGetSize(pTableMetaInfo->vgroupIdList); + for (int32_t i = 0; i < numOfSubs; ++i) { (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8fb3df94d5844734ec07ee83e50d193fc0e7c7fa..111ebc4e8c690cef9c52ea214dacde0cf5d58dfd 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -35,7 +35,6 @@ SRpcIpSet tscDnodeIpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); -void (*tscUpdateVnodeMsg[TSDB_SQL_MAX])(SSqlObj *pSql, char *buf); void tscProcessActivityTimer(void *handle, void *tmrId); int tscKeepConn[TSDB_SQL_MAX] = {0}; TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid); @@ -188,7 +187,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .pCont = pMsg, .contLen = pSql->cmd.payloadLen, .handle = pSql, - .code = 0 + .code = 0 }; rpcSendRequest(pVnodeConn, pSql->ipList, &rpcMsg); } else { @@ -209,7 +208,6 @@ int tscSendMsgToServer(SSqlObj *pSql) { } void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { - tscTrace("response:%s is received, len:%d error:%s", taosMsg[rpcMsg->msgType], rpcMsg->contLen, tstrerror(rpcMsg->code)); SSqlObj *pSql = (SSqlObj *)rpcMsg->handle; if (pSql == NULL || pSql->signature != pSql) { tscError("%p sql is already released, signature:%p", pSql, pSql->signature); @@ -256,7 +254,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { rpcFreeCont(rpcMsg->pCont); return; } else { - tscTrace("%p it shall renew meter meta, code:%d", pSql, rpcMsg->code); + tscTrace("%p it shall renew meter meta, code:%d", pSql, tstrerror(rpcMsg->code)); pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; pSql->res.code = rpcMsg->code; // keep the previous error code @@ -273,19 +271,19 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { } pSql->retry = 0; - pRes->rspLen = 0; + if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_NETWORK_UNAVAIL; } else { - tscTrace("%p query is cancelled, code:%d", pSql, pRes->code); + tscTrace("%p query is cancelled, code:%d", pSql, tstrerror(pRes->code)); } if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { assert(rpcMsg->msgType == pCmd->msgType + 1); - pRes->code = (int32_t)rpcMsg->code; + pRes->code = rpcMsg->code; pRes->rspType = rpcMsg->msgType; - pRes->rspLen = rpcMsg->contLen; + pRes->rspLen = rpcMsg->contLen; char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); if (tmp == NULL) { @@ -316,21 +314,20 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { pRes->numOfRows += pMsg->affectedRows; tscTrace("%p cmd:%d code:%d, inserted rows:%d, rsp len:%d", pSql, pCmd->command, pRes->code, - *(int32_t *)pRes->pRsp, pRes->rspLen); + pMsg->affectedRows, pRes->rspLen); } else { - tscTrace("%p cmd:%d code:%d rsp len:%d", pSql, pCmd->command, pRes->code, pRes->rspLen); + tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen); } } if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); - + if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { - int command = pCmd->command; - void *taosres = tscKeepConn[command] ? pSql : NULL; + void *taosres = tscKeepConn[pCmd->command] ? pSql : NULL; rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows; - - tscTrace("%p Async SQL result:%d res:%p", pSql, rpcMsg->code, taosres); + + tscTrace("%p Async SQL result:%s res:%p", pSql, tstrerror(pRes->code), taosres); /* * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj @@ -344,14 +341,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { (*pSql->fp)(pSql->param, taosres, rpcMsg->code); if (shouldFree) { - // If it is failed, all objects allocated during execution taos_connect_a should be released - if (command == TSDB_SQL_CONNECT) { - taos_close(pObj); - tscTrace("%p Async sql close failed connection", pSql); - } else { - tscFreeSqlObj(pSql); - tscTrace("%p Async sql is automatically freed", pSql); - } + tscFreeSqlObj(pSql); + tscTrace("%p Async sql is automatically freed", pSql); } } @@ -369,7 +360,7 @@ int doProcessSql(SSqlObj *pSql) { pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || pCmd->command == TSDB_SQL_META || - pCmd->command == TSDB_SQL_METRIC) { + pCmd->command == TSDB_SQL_STABLEVGROUP) { tscBuildMsg[pCmd->command](pSql, NULL); } @@ -389,7 +380,7 @@ int tscProcessSql(SSqlObj *pSql) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = NULL; - uint16_t type = 0; + uint16_t type = 0; if (pQueryInfo != NULL) { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -508,76 +499,44 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pRetrieveMsg->free = htons(pQueryInfo->type); pMsg += sizeof(pQueryInfo->type); - pSql->cmd.payloadLen = pMsg - pStart; + pRetrieveMsg->header.vgId = htonl(1); + pMsg += sizeof(SRetrieveTableMsg); + + pRetrieveMsg->header.contLen = htonl(pSql->cmd.payloadLen); + pSql->cmd.msgType = TSDB_MSG_TYPE_RETRIEVE; - return TSDB_CODE_SUCCESS; } -void tscUpdateVnodeInSubmitMsg(SSqlObj *pSql, char *buf) { - //SShellSubmitMsg *pShellMsg; - //char * pMsg; - //STableMetaInfo * pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); - - //STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; - - //pMsg = buf + tsRpcHeadSize; - - //TODO set iplist - //pShellMsg = (SShellSubmitMsg *)pMsg; - //pShellMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode); - //tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pSql->index].ip), - // htons(pShellMsg->vnode)); -} - int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SShellSubmitMsg *pShellMsg; - char * pMsg, *pStart; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - pStart = pSql->cmd.payload + tsRpcHeadSize; - pMsg = pStart; - - pShellMsg = (SShellSubmitMsg *)pMsg; + char* pMsg = pSql->cmd.payload + tsRpcHeadSize; + + // NOTE: shell message size should not include SMsgDesc + int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc); - pShellMsg->desc.numOfVnodes = htonl(1); + SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg; + + pMsgDesc->numOfVnodes = htonl(1); //todo set the right number of vnodes + pMsg += sizeof(SMsgDesc); + + SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg; - pShellMsg->import = htons(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT) ? 0 : 1); pShellMsg->header.vgId = htonl(pTableMeta->vgId); - pShellMsg->header.contLen = htonl(pSql->cmd.payloadLen); + pShellMsg->header.contLen = htonl(size); + pShellMsg->length = pShellMsg->header.contLen; - pShellMsg->numOfTables = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted + pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted - // pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here + // pSql->cmd.payloadLen is set during copying data into paylaod pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; -// tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip), -// htons(pShellMsg->vnode)); - -// pSql->cmd.payloadLen = sizeof(SShellSubmitMsg); - + tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htons(pMsgDesc->numOfVnodes)); + return TSDB_CODE_SUCCESS; } -void tscUpdateVnodeInQueryMsg(SSqlObj *pSql, char *buf) { - //TODO -// SSqlCmd * pCmd = &pSql->cmd; -// STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); -// -// char * pStart = buf + tsRpcHeadSize; -// SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; -// -// if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // pColumnModel == NULL, query on meter -// STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; -// pQueryMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode); -// } else { // query on metric -// SSuperTableMeta * pMetricMeta = pTableMetaInfo->pMetricMeta; -// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); -// pQueryMsg->vnode = htons(pVnodeSidList->vpeerDesc[pSql->index].vnode); -// } -} - /* * for meter query, simply return the size <= 1k * for metric query, estimate size according to meter tags @@ -595,7 +554,10 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize; } - + + int32_t size = 4096; + +#if 0 SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); @@ -606,45 +568,23 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { if (pQueryInfo->tsBuf != NULL) { size += pQueryInfo->tsBuf->fileSize; } - +#endif + return size; } -static char *doSerializeTableInfo(SSqlObj *pSql, int32_t numOfTables, int32_t vgId, char *pMsg) { +static char *doSerializeTableInfo(SSqlObj *pSql, int32_t vgId, char *pMsg) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; - - tscTrace("%p vgId:%d, query on %d tables", pSql, vgId, numOfTables); - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { -#ifdef _DEBUG_VIEW - tscTrace("%p sid:%d, uid:%" PRIu64, pSql, pTableMetaInfo->pTableMeta->sid, pTableMetaInfo->pTableMeta->uid); -#endif - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->sid = htonl(pTableMeta->sid); - pTableIdInfo->uid = htobe64(pTableMeta->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); - pMsg += sizeof(STableIdInfo); - } else { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); - - for (int32_t i = 0; i < numOfTables; ++i) { - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - STableIdInfo *pQueryMeterInfo = tscGetMeterSidInfo(pVnodeSidList, i); - - pTableIdInfo->sid = htonl(pQueryMeterInfo->sid); - pTableIdInfo->uid = htobe64(pQueryMeterInfo->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pQueryMeterInfo->uid)); - - pMsg += sizeof(STableIdInfo); - -#ifdef _DEBUG_VIEW - tscTrace("%p sid:%d, uid:%" PRId64, pSql, pQueryMeterInfo->sid, pQueryMeterInfo->uid); -#endif - } - } - + tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, vgId, pTableMetaInfo->name, pTableMeta->uid); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->sid = htonl(pTableMeta->sid); + pTableIdInfo->uid = htobe64(pTableMeta->uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); + + pMsg += sizeof(STableIdInfo); return pMsg; } @@ -657,14 +597,27 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscError("%p failed to malloc for query msg", pSql); return -1; } - + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - char *pStart = pCmd->payload + tsRpcHeadSize; + if (pQueryInfo->colList.numOfCols <= 0) { + tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); + return -1; + } + + if (pQueryInfo->intervalTime < 0) { + tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); + return -1; + } + + if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { + tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); + return -1; + } - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; + char *pStart = pCmd->payload + tsRpcHeadSize; SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; @@ -673,18 +626,17 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { numOfTables = 1; - - pQueryMsg->uid = pTableMeta->uid; - pQueryMsg->numOfTagsCols = 0; - - pQueryMsg->vgId = htonl(pTableMeta->vgId); + pQueryMsg->head.vgId = htonl(pTableMeta->vgId); tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name); } else { // query on super table if (pTableMetaInfo->vnodeIndex < 0) { tscError("%p error vnodeIdx:%d", pSql, pTableMetaInfo->vnodeIndex); return -1; } - + + uint32_t vnodeId = 1; + +#if 0 SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); uint32_t vnodeId = pVnodeSidList->vpeerDesc[pVnodeSidList->index].vnode; @@ -693,14 +645,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscError("%p vid:%d,error numOfTables in query message:%d", pSql, vnodeId, numOfTables); return -1; // error } - +#endif + tscTrace("%p query on vid:%d, number of tables:%d", pSql, vnodeId, numOfTables); - pQueryMsg->vgId = htons(vnodeId); + pQueryMsg->head.vgId = htonl(vnodeId); + numOfTables = 1; } - pQueryMsg->numOfTables = htonl(numOfTables); - pQueryMsg->numOfTagsCols = htons(pTableMetaInfo->numOfTags); - if (pQueryInfo->order.order == TSQL_SO_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->stime); pQueryMsg->window.ekey = htobe64(pQueryInfo->etime); @@ -709,54 +660,29 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->window.ekey = htobe64(pQueryInfo->stime); } - pQueryMsg->order = htons(pQueryInfo->order.order); - pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId); - - pQueryMsg->interpoType = htons(pQueryInfo->interpoType); - - pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); - pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); - - pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols); - - if (pQueryInfo->colList.numOfCols <= 0) { - tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); - return -1; - } - - pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); + pQueryMsg->numOfTables = htonl(numOfTables); + pQueryMsg->order = htons(pQueryInfo->order.order); + pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId); + pQueryMsg->interpoType = htons(pQueryInfo->interpoType); + pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); + pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); + pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols); + pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); + pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; - pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); - - if (pQueryInfo->intervalTime < 0) { - tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); - return -1; - } - - if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { - tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); - return -1; - } - pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // query on meter - pQueryMsg->tagLength = 0; - } else { // query on super table - pQueryMsg->tagLength = htons(pMetricMeta->tagLen); - } - pQueryMsg->queryType = htons(pQueryInfo->type); pQueryMsg->numOfOutputCols = htons(pQueryInfo->exprsInfo.numOfExprs); - if (pQueryInfo->fieldsInfo.numOfOutputCols < 0) { - tscError("%p illegal value of number of output columns in query msg: %d", pSql, - pQueryInfo->fieldsInfo.numOfOutputCols); + int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutputCols; + if (numOfOutput < 0) { + tscError("%p illegal value of number of output columns in query msg: %d", pSql, numOfOutput); return -1; } // set column list ids - char * pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); + char *pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); SSchema *pSchema = tscGetTableSchema(pTableMeta); for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { @@ -774,7 +700,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->colList[i].colId = htons(pColSchema->colId); pQueryMsg->colList[i].bytes = htons(pColSchema->bytes); - pQueryMsg->colList[i].type = htons(pColSchema->type); + pQueryMsg->colList[i].type = htons(pColSchema->type); pQueryMsg->colList[i].numOfFilters = htons(pCol->numOfFilters); // append the filter information after the basic column information @@ -808,7 +734,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { bool hasArithmeticFunction = false; SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; - for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); @@ -822,11 +747,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return -1; } - pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); + pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); pSqlFuncExpr->colInfo.colIdx = htons(pExpr->colInfo.colIdx); - pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); + pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); - pSqlFuncExpr->functionId = htons(pExpr->functionId); + pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); pMsg += sizeof(SSqlFuncExprMsg); @@ -864,25 +789,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->colNameLen = htonl(len); // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->vgId), pMsg); - - // only include the required tag column schema. If a tag is not required, it won't be sent to vnode - if (pTableMetaInfo->numOfTags > 0) { - // always transfer tag schema to vnode if exists - SSchema *pTagSchema = tscGetTableTagSchema(pTableMeta); - - for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { - if (pTableMetaInfo->tagColumnIndex[j] == TSDB_TBNAME_COLUMN_INDEX) { - SSchema tbSchema = { - .bytes = TSDB_TABLE_NAME_LEN, .colId = TSDB_TBNAME_COLUMN_INDEX, .type = TSDB_DATA_TYPE_BINARY}; - memcpy(pMsg, &tbSchema, sizeof(SSchema)); - } else { - memcpy(pMsg, &pTagSchema[pTableMetaInfo->tagColumnIndex[j]], sizeof(SSchema)); - } - - pMsg += sizeof(SSchema); - } - } + pMsg = doSerializeTableInfo(pSql, htons(pQueryMsg->head.vgId), pMsg); SSqlGroupbyExpr *pGroupbyExpr = &pQueryInfo->groupbyExpr; if (pGroupbyExpr->numOfGroupCols != 0) { @@ -940,14 +847,42 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); } + // serialize tag column query condition + if (pQueryInfo->tagCond.numOfTagCond > 0) { + STagCond* pTagCond = &pQueryInfo->tagCond; + + SCond *pCond = tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid); + if (pCond != NULL && pCond->cond != NULL) { + size_t condLen = strlen(pCond->cond) + 1; + + bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); + if (!ret) { + tscError("%p mbs to ucs4 failed:%d", pSql, tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid)); + return 0; + } + + pQueryMsg->tagCondLen = htons(condLen); + pMsg += condLen * TSDB_NCHAR_SIZE; + } + } + + // tbname in/like query expression should be sent to mgmt node + STagCond* pTagCond = &pQueryInfo->tagCond; + if (pTagCond->tbnameCond.cond != NULL) { + size_t s = strlen(pTagCond->tbnameCond.cond); + memcpy(pMsg, pTagCond->tbnameCond.cond, s); + + pQueryMsg->nameCondLen = htons(s); + pMsg += s; + } + msgLen = pMsg - pStart; tscTrace("%p msg built success,len:%d bytes", pSql, msgLen); pCmd->payloadLen = msgLen; pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; - pQueryMsg->contLen = htonl(msgLen); - + pQueryMsg->head.contLen = htonl(msgLen); assert(msgLen + minMsgSize() <= size); return TSDB_CODE_SUCCESS; @@ -1478,11 +1413,15 @@ int tscProcessTagRetrieveRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int32_t numOfRes = 0; +#if 0 if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_TAGPRJ) { numOfRes = pTableMetaInfo->pMetricMeta->numOfTables; } else { numOfRes = 1; // for count function, there is only one output. } + +#endif + return tscLocalResultCommonBuilder(pSql, numOfRes); } @@ -1568,8 +1507,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(STagData); } - msgLen = pMsg - (char*)pInfoMsg; - pCmd->payloadLen = msgLen; + pCmd->payloadLen = pMsg - (char*)pInfoMsg;; pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META; tfree(tmpData); @@ -1643,7 +1581,9 @@ static int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); } -int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { +int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + +#if 0 SSuperTableMetaMsg *pMetaMsg; char * pMsg, *pStart; int msgLen = 0; @@ -1706,13 +1646,13 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // convert to unicode before sending to mnode for metric query int32_t condLen = 0; if (pTagCond->numOfTagCond > 0) { - SCond *pCond = tsGetMetricQueryCondPos(pTagCond, uid); + SCond *pCond = tsGetSTableQueryCondPos(pTagCond, uid); if (pCond != NULL && pCond->cond != NULL) { condLen = strlen(pCond->cond) + 1; bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); if (!ret) { - tscError("%p mbs to ucs4 failed:%s", pSql, tsGetMetricQueryCondPos(pTagCond, uid)); + tscError("%p mbs to ucs4 failed:%s", pSql, tsGetSTableQueryCondPos(pTagCond, uid)); return 0; } } @@ -1782,9 +1722,20 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { msgLen = pMsg - pStart; pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_META; + pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; assert(msgLen + minMsgSize() <= size); +#endif + SSqlCmd *pCmd = &pSql->cmd; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + + SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pCmd->payload; + strncpy(pStableVgroupMsg->tableId, pTableMetaInfo->name, tListLen(pStableVgroupMsg->tableId)); + + pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; + pCmd->payloadLen = sizeof(SCMSTableVgroupMsg); + return TSDB_CODE_SUCCESS; } @@ -1853,14 +1804,13 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->vgId = htonl(pMetaMsg->vgId); pMetaMsg->uid = htobe64(pMetaMsg->uid); pMetaMsg->contLen = htons(pMetaMsg->contLen); + pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if (pMetaMsg->sid < 0 || pMetaMsg->vgId < 0) { tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgId, pMetaMsg->sid); return TSDB_CODE_INVALID_VALUE; } - pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - if (pMetaMsg->numOfTags > TSDB_MAX_TAGS || pMetaMsg->numOfTags < 0) { tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); return TSDB_CODE_INVALID_VALUE; @@ -1884,23 +1834,20 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pSchema++; } -// rsp += numOfTotalCols * sizeof(SSchema); -// -// int32_t tagLen = 0; -// SSchema *pTagsSchema = tscGetTableTagSchema(pMetaMsg); -// -// if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { -// for (int32_t i = 0; i < pMetaMsg->numOfTags; ++i) { -// tagLen += pTagsSchema[i].bytes; -// } -// } -// -// rsp += tagLen; -// int32_t size = (int32_t)(rsp - (char *)pMetaMsg); - size_t size = 0; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); +#if 0 + // if current table is created according to super table, get the table meta of super table + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + char id[TSDB_TABLE_ID_LEN + 1] = {0}; + strncpy(id, pMetaMsg->stableId, TSDB_TABLE_ID_LEN); + + // NOTE: if the table meta of super table is not cached at client side yet, the pSTable is NULL + pTableMeta->pSTable = taosCacheAcquireByName(tscCacheHandle, id); + } +#endif + // todo add one more function: taosAddDataIfNotExists(); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); @@ -1914,6 +1861,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { } free(pTableMeta); + return TSDB_CODE_SUCCESS; } @@ -2027,22 +1975,11 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; } -int tscProcessMetricMetaRsp(SSqlObj *pSql) { - SSuperTableMeta *pMeta; - uint8_t ieType; +int tscProcessSTableVgroupRsp(SSqlObj *pSql) { +#if 0 void ** metricMetaList = NULL; int32_t * sizes = NULL; - - char *rsp = pSql->res.pRsp; - - ieType = *rsp; - if (ieType != TSDB_IE_TYPE_META) { - tscError("invalid ie type:%d", ieType); - return TSDB_CODE_INVALID_IE; - } - - rsp++; - + int32_t num = htons(*(int16_t *)rsp); rsp += sizeof(int16_t); @@ -2126,7 +2063,6 @@ int tscProcessMetricMetaRsp(SSqlObj *pSql) { // release the used metricmeta taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); - pTableMetaInfo->pMetricMeta = (SSuperTableMeta *)taosCachePut(tscCacheHandle, name, (char *)metricMetaList[i], sizes[i], tsMetricMetaKeepTimer); tfree(metricMetaList[i]); @@ -2146,7 +2082,23 @@ _error_clean: free(sizes); free(metricMetaList); - +#endif + + SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pSql->res.pRsp; + pStableVgroup->numOfDnodes = htonl(pStableVgroup->numOfDnodes); + + SSqlObj* pparent = pSql->param; + assert(pparent != NULL); + + SSqlCmd* pCmd = &pparent->cmd; + STableMetaInfo* pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + pInfo->vgroupIdList = taosArrayInit(pStableVgroup->numOfDnodes, sizeof(int32_t)); + + // todo opt performance + for(int32_t i = 0; i < pStableVgroup->numOfDnodes; ++i) { + taosArrayPush(pInfo->vgroupIdList, &pStableVgroup->dnodeIps[i]); + } + return pSql->res.code; } @@ -2272,7 +2224,7 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { if (pTableMetaInfo->pTableMeta) { taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); +// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); } return 0; @@ -2293,7 +2245,7 @@ int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { bool isSuperTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo); taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); +// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); if (isSuperTable) { // if it is a super table, reset whole query cache tscTrace("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); @@ -2329,9 +2281,10 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { pRes->numOfRows = htonl(pRetrieve->numOfRows); pRes->precision = htons(pRetrieve->precision); - pRes->offset = htobe64(pRetrieve->offset); - pRes->useconds = htobe64(pRetrieve->useconds); - pRes->data = pRetrieve->data; + pRes->offset = htobe64(pRetrieve->offset); + pRes->useconds = htobe64(pRetrieve->useconds); + pRes->completed = (pRetrieve->completed == 1); + pRes->data = pRetrieve->data; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); tscSetResultPointer(pQueryInfo, pRes); @@ -2356,7 +2309,6 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { } pRes->row = 0; - tscTrace("%p numOfRows:%d, offset:%d", pSql, pRes->numOfRows, pRes->offset); return 0; @@ -2379,7 +2331,7 @@ int tscProcessRetrieveRspFromLocal(SSqlObj *pSql) { void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); -static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { +static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { tscError("%p malloc failed for new sqlobj to get meter meta", pSql); @@ -2406,7 +2358,7 @@ static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMet STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - strcpy(pNewMeterMetaInfo->name, pTableMetaInfo->name); + strncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, tListLen(pNewMeterMetaInfo->name)); memcpy(pNew->cmd.payload, pSql->cmd.payload, TSDB_DEFAULT_PAYLOAD_SIZE); // tag information if table does not exists. tscTrace("%p new pSqlObj:%p to get tableMeta", pSql, pNew); @@ -2424,7 +2376,7 @@ static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMet int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { assert(strlen(pTableMetaInfo->name) != 0); - // If this STableMetaInfo owns a metermeta, release it first + // If this STableMetaInfo owns a table meta, release it first if (pTableMetaInfo->pTableMeta != NULL) { taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false); } @@ -2437,12 +2389,8 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { return TSDB_CODE_SUCCESS; } - - /* - * for async insert operation, release data block buffer before issue new object to get metermeta - * because in table meta callback function, the tscParse function will generate the submit data blocks - */ - return doGetMeterMetaFromServer(pSql, pTableMetaInfo); + + return getTableMetaFromMgmt(pSql, pTableMetaInfo); } int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) { @@ -2491,7 +2439,7 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { tscWaitingForCreateTable(pCmd); taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); - code = doGetMeterMetaFromServer(pSql, pTableMetaInfo); // todo ?? + code = getTableMetaFromMgmt(pSql, pTableMetaInfo); // todo ?? } else { tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, tscGetNumOfTags(pTableMetaInfo->pTableMeta), pCmd->numOfCols, pTableMetaInfo->pTableMeta->uid, @@ -2501,30 +2449,34 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { return code; } -int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { +int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { int code = TSDB_CODE_NETWORK_UNAVAIL; SSqlCmd *pCmd = &pSql->cmd; - /* - * the query condition is serialized into pCmd->payload, we need to rebuild key for metricmeta info in cache. - */ - bool required = false; + //the query condition is serialized into pCmd->payload, we need to rebuild key for stable meta info in cache. + bool required = false; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); + if (pQueryInfo->pTableMetaInfo[0]->vgroupIdList != NULL) { + return TSDB_CODE_SUCCESS; + } + +#if 0 + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0}; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pTableMetaInfo->pTableMeta->uid); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); +// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); SSuperTableMeta *ppMeta = (SSuperTableMeta *)taosCacheAcquireByName(tscCacheHandle, tagstr); if (ppMeta == NULL) { required = true; break; } else { - pTableMetaInfo->pMetricMeta = ppMeta; +// pTableMetaInfo->pMetricMeta = ppMeta; } } @@ -2532,12 +2484,13 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { if (!required) { return TSDB_CODE_SUCCESS; } - +#endif + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_METRIC; + pNew->cmd.command = TSDB_SQL_STABLEVGROUP; SQueryInfo *pNewQueryInfo = NULL; if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) { @@ -2574,7 +2527,7 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { // tscFreeSubqueryInfo(pCmd); // } - tscTrace("%p allocate new pSqlObj:%p to get metricMeta", pSql, pNew); + tscTrace("%p allocate new pSqlObj:%p to get stable vgroupInfo", pSql, pNew); pNew->fp = tscTableMetaCallBack; pNew->param = pSql; code = tscProcessSql(pNew); @@ -2611,7 +2564,7 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg; tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg; - tscBuildMsg[TSDB_SQL_METRIC] = tscBuildMetricMetaMsg; + tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; tscBuildMsg[TSDB_SQL_MULTI_META] = tscBuildMultiMeterMetaMsg; tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; @@ -2629,7 +2582,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_CONNECT] = tscProcessConnectRsp; tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; - tscProcessMsgRsp[TSDB_SQL_METRIC] = tscProcessMetricMetaRsp; + tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; @@ -2655,7 +2608,4 @@ void tscInitMsgsFp() { tscKeepConn[TSDB_SQL_SELECT] = 1; tscKeepConn[TSDB_SQL_FETCH] = 1; tscKeepConn[TSDB_SQL_HB] = 1; - - tscUpdateVnodeMsg[TSDB_SQL_SELECT] = tscUpdateVnodeInQueryMsg; - tscUpdateVnodeMsg[TSDB_SQL_INSERT] = tscUpdateVnodeInSubmitMsg; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 5d93e44c7712910c8a34805def9dfe9c2ed0aa82..e4b07a0f645e09892c720899d9be6518c462f844 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -147,7 +147,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con } // tsRpcHeaderSize will be updated during RPC initialization, so only after it initialization, this value is valid - tsInsertHeadSize = tsRpcHeadSize + sizeof(SShellSubmitMsg); + tsInsertHeadSize = tsRpcHeadSize + sizeof(SMsgDesc) + sizeof(SSubmitMsg); return pObj; } @@ -546,11 +546,11 @@ static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) { * if the global limitation is not reached, and current result has not exhausted, or next more vnodes are * available, goes on */ - if (pMetaInfo->vnodeIndex < pMetaInfo->pMetricMeta->numOfVnodes && pRes1->row < pRes1->numOfRows && - (!tscHasReachLimitation(pQueryInfo1, pRes1))) { - allSubqueryExhausted = false; - break; - } +// if (pMetaInfo->vnodeIndex < pMetaInfo->pMetricMeta->numOfVnodes && pRes1->row < pRes1->numOfRows && +// (!tscHasReachLimitation(pQueryInfo1, pRes1))) { +// allSubqueryExhausted = false; +// break; +// } } hasData = !allSubqueryExhausted; @@ -594,11 +594,7 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) { } if (numOfTableHasRes >= 2) { // do merge result - success = (doSetResultRowData(pSql->pSubs[0]) != NULL) && (doSetResultRowData(pSql->pSubs[1]) != NULL); - // TSKEY key1 = *(TSKEY *)pRes1->tsrow[0]; - // TSKEY key2 = *(TSKEY *)pRes2->tsrow[0]; - // printf("first:%" PRId64 ", second:%" PRId64 "\n", key1, key2); } else { // only one subquery SSqlObj *pSub = pSql->pSubs[0]; if (pSub == NULL) { @@ -653,66 +649,9 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) { return pRes->tsrow; } -TAOS_ROW taos_fetch_row_impl(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - return NULL; - } - - if (pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) { - tscFetchDatablockFromSubquery(pSql); - - if (pRes->code == TSDB_CODE_SUCCESS) { - tscTrace("%p data from all subqueries have been retrieved to client", pSql); - return tscBuildResFromSubqueries(pSql); - } else { - tscTrace("%p retrieve data from subquery failed, code:%d", pSql, pRes->code); - return NULL; - } - - } else if (pRes->row >= pRes->numOfRows) { - /** - * NOT a join query - * - * If the data block of current result set have been consumed already, try fetch next result - * data block from virtual node. - */ - tscResetForNextRetrieve(pRes); - - if (pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - } - - tscProcessSql(pSql); // retrieve data from virtual node - - // if failed to retrieve data from current virtual node, try next one if exists - if (hasMoreVnodesToTry(pSql)) { - tscTryQueryNextVnode(pSql, NULL); - } - - /* - * local reducer has handle this case, - * so no need to add the pRes->numOfRows for super table query - */ - if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC) { - pRes->numOfTotalInCurrentClause += pRes->numOfRows; - } - - if (pRes->numOfRows == 0) { - return NULL; - } - } - - return doSetResultRowData(pSql); -} - static void asyncFetchCallback(void *param, TAOS_RES *tres, int numOfRows) { SSqlObj* pSql = (SSqlObj*) tres; - if (numOfRows < 0) { - // set the error code + if (numOfRows < 0) { // set the error code pSql->res.code = -numOfRows; } @@ -729,12 +668,14 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT) { + if (pRes->qhandle == 0 || + pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + pCmd->command == TSDB_SQL_INSERT) { return NULL; } // current data are exhausted, fetch more data - if (pRes->data == NULL || (pRes->data != NULL && pRes->row >= pRes->numOfRows && + if (pRes->data == NULL || (pRes->data != NULL && pRes->row >= pRes->numOfRows && pRes->completed != true && (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_METRIC || pCmd->command == TSDB_SQL_FETCH))) { taos_fetch_rows_a(res, asyncFetchCallback, pSql->pTscObj); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 46e3ac2e60b9e28fa21251c0fbe4a2db3fdd0351..4fadad5021b5841f482f365f9a6d07ea6777ad9b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -80,7 +80,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; if (code == 0 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - code = tscGetMetricMeta(pSql, 0); + code = tscGetSTableVgroupInfo(pSql, 0); pSql->res.code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; @@ -504,7 +504,6 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p } tsem_init(&pSql->rspSem, 0, 0); - tsem_init(&pSql->emptyRspSem, 0, 1); SSqlInfo SQLInfo = {0}; tSQLParse(&SQLInfo, pSql->sqlstr); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 014b0c5cb7fabf70a7d48918fe600a9936536cfd..616bcbba504df2d949a4cbb3aee7ef84b7f6e082 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -178,11 +178,11 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); int numOfTables = 0; if (!UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { - SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; - for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); - numOfTables += pVnodeSidList->numOfSids; - } +// SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; +// for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { +// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); +// numOfTables += pVnodeSidList->numOfSids; +// } } SSubscriptionProgress* progress = (SSubscriptionProgress*)calloc(numOfTables, sizeof(SSubscriptionProgress)); @@ -197,17 +197,17 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { progress[0].uid = uid; progress[0].key = tscGetSubscriptionProgress(pSub, uid); } else { - SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; - numOfTables = 0; - for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); - for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) { - STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j); - int64_t uid = pTableMetaInfo->uid; - progress[numOfTables].uid = uid; - progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid); - } - } +// SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; +// numOfTables = 0; +// for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { +// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); +// for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) { +// STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j); +// int64_t uid = pTableMetaInfo->uid; +// progress[numOfTables].uid = uid; +// progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid); +// } +// } qsort(progress, numOfTables, sizeof(SSubscriptionProgress), tscCompareSubscriptionProgress); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1df977d99626cec34ac3750eb668244dd06c33d4..b0c7b68ab4cfc263f6728d4ff8e95c224ec988cf 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -476,7 +476,8 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { assert(pQueryInfo->numOfTables == 1); // for projection query, need to try next vnode - int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; +// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; + int32_t totalVnode = 0; if ((++pTableMetaInfo->vnodeIndex) < totalVnode) { tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, pTableMetaInfo->vnodeIndex - 1, pTableMetaInfo->vnodeIndex, totalVnode, pRes->numOfTotal); @@ -541,16 +542,16 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { assert(pQueryInfo->numOfTables == 1); // for projection query, need to try next vnode if current vnode is exhausted - if ((++pTableMetaInfo->vnodeIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) { - pSupporter->pState->numOfCompleted = 0; - pSupporter->pState->numOfTotal = 1; - - pSql->cmd.command = TSDB_SQL_SELECT; - pSql->fp = tscJoinQueryCallback; - tscProcessSql(pSql); - - return; - } +// if ((++pTableMetaInfo->vnodeIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) { +// pSupporter->pState->numOfCompleted = 0; +// pSupporter->pState->numOfTotal = 1; +// +// pSql->cmd.command = TSDB_SQL_SELECT; +// pSql->fp = tscJoinQueryCallback; +// tscProcessSql(pSql); +// +// return; +// } } int32_t numOfTotal = pSupporter->pState->numOfTotal; @@ -608,10 +609,10 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes && - (!tscHasReachLimitation(pQueryInfo, pRes))) { - numOfFetch++; - } +// if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes && +// (!tscHasReachLimitation(pQueryInfo, pRes))) { +// numOfFetch++; +// } } else { if (pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pQueryInfo, pRes))) { numOfFetch++; @@ -788,7 +789,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { * data instead of returning to its invoker */ if (pTableMetaInfo->vnodeIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - assert(pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes); +// assert(pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes); pSupporter->pState->numOfCompleted = 0; // reset the record value pSql->fp = joinRetrieveCallback; // continue retrieve data @@ -1005,11 +1006,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pRes->qhandle = 1; // hack the qhandle check - const uint32_t nBufferSize = (1 << 16); // 64KB + const uint32_t nBufferSize = (1u << 16); // 64KB SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes; + + int32_t numOfSubQueries = taosArrayGetSize(pTableMetaInfo->vgroupIdList); assert(numOfSubQueries > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); @@ -1115,7 +1117,7 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { tfree(trsupport); } -static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); +static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) { // set no disk space error info @@ -1137,7 +1139,7 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES pthread_mutex_unlock(&trsupport->queryMutex); - tscRetrieveFromVnodeCallBack(trsupport, tres, trsupport->pState->code); + tscRetrieveFromDnodeCallBack(trsupport, tres, trsupport->pState->code); } static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -1232,7 +1234,86 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq } } -static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { +static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { + int32_t idx = trsupport->subqueryIndex; + SSqlObj * pPObj = trsupport->pParentSqlObj; + tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; + + SSubqueryState* pState = trsupport->pState; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + + // data in from current vnode is stored in cache and disk + uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; +// tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, +// pSvd->vnode, numOfRowsFromSubquery, idx); + + tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); + +#ifdef _DEBUG_VIEW + printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems); + SSrcColumnInfo colInfo[256] = {0}; + tscGetSrcColumnInfo(colInfo, pQueryInfo); + tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems, + trsupport->localBuffer->numOfElems, colInfo); +#endif + + if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { + tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, + tsAvailTmpDirGB, tsMinimalTmpDirGB); + tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE); + return; + } + + // each result for a vnode is ordered as an independant list, + // then used as an input of loser tree for disk-based merge routine + int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, + pQueryInfo->groupbyExpr.orderType); + if (ret != 0) { // set no disk space error info, and abort retry + return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE); + } + + // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion + // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. + // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. + int32_t numOfTotal = pState->numOfTotal; + + int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); + if (finished < numOfTotal) { + tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); + return tscFreeSubSqlObj(trsupport, pSql); + } + + // all sub-queries are returned, start to local merge process + pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; + + tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, + pState->numOfTotal, pState->numOfRetrievedRows); + + SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); + tscClearInterpInfo(pPQueryInfo); + + tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, + &pPObj->cmd, &pPObj->res); + tscTrace("%p build loser tree completed", pPObj); + + pPObj->res.precision = pSql->res.precision; + pPObj->res.numOfRows = 0; + pPObj->res.row = 0; + + // only free once + tfree(trsupport->pState); + tscFreeSubSqlObj(trsupport, pSql); + + // set the command flag must be after the semaphore been correctly set. + pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; + if (pPObj->res.code == TSDB_CODE_SUCCESS) { + (*pPObj->fp)(pPObj->param, pPObj, 0); + } else { + tscQueueAsyncRes(pPObj); + } +} + +static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { SRetrieveSupport *trsupport = (SRetrieveSupport *)param; int32_t idx = trsupport->subqueryIndex; SSqlObj * pPObj = trsupport->pParentSqlObj; @@ -1260,15 +1341,16 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); - SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; +// SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); +// SVnodeSidList *vnodeInfo = 0; +// SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; if (numOfRows > 0) { assert(pRes->numOfRows == numOfRows); int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, - pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); +// tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, +// pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64, @@ -1294,85 +1376,16 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data, pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); - if (ret < 0) { - // set no disk space error info, and abort retry + if (ret < 0) { // set no disk space error info, and abort retry tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - } else { + } else if (pRes->completed) { + tscAllDataRetrievedFromDnode(trsupport, pSql); + } else { // continue fetch data from dnode pthread_mutex_unlock(&trsupport->queryMutex); - taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); - } - - } else { // all data has been retrieved to client - /* data in from current vnode is stored in cache and disk */ - uint32_t numOfRowsFromVnode = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; - tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, - pSvd->vnode, numOfRowsFromVnode, idx); - - tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); - -#ifdef _DEBUG_VIEW - printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems); - SSrcColumnInfo colInfo[256] = {0}; - tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems, - trsupport->localBuffer->numOfElems, colInfo); -#endif - - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, - tsAvailTmpDirGB, tsMinimalTmpDirGB); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - return; - } - - // each result for a vnode is ordered as an independant list, - // then used as an input of loser tree for disk-based merge routine - int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, - pQueryInfo->groupbyExpr.orderType); - if (ret != 0) { - /* set no disk space error info, and abort retry */ - return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - } - - // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion - // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. - // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. - int32_t numOfTotal = pState->numOfTotal; - - int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); - if (finished < numOfTotal) { - tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); - return tscFreeSubSqlObj(trsupport, pSql); - } - - // all sub-queries are returned, start to local merge process - pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; - - tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, - pState->numOfTotal, pState->numOfRetrievedRows); - - SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); - tscClearInterpInfo(pPQueryInfo); - - tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, - &pPObj->cmd, &pPObj->res); - tscTrace("%p build loser tree completed", pPObj); - - pPObj->res.precision = pSql->res.precision; - pPObj->res.numOfRows = 0; - pPObj->res.row = 0; - - // only free once - tfree(trsupport->pState); - tscFreeSubSqlObj(trsupport, pSql); - - // set the command flag must be after the semaphore been correctly set. - pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - if (pPObj->res.code == TSDB_CODE_SUCCESS) { - (*pPObj->fp)(pPObj->param, pPObj, 0); - } else { - tscQueueAsyncRes(pPObj); + taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } + } else { // all data has been retrieved to client + tscAllDataRetrievedFromDnode(trsupport, pSql); } } @@ -1409,10 +1422,10 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SVnodeSidList *vnodeInfo = NULL; SVnodeDesc * pSvd = NULL; - if (pTableMetaInfo->pMetricMeta != NULL) { - vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); - pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; - } +// if (pTableMetaInfo->pMetricMeta != NULL) { +// vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); +// pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; +// } SSubqueryState* pState = trsupport->pState; assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 && @@ -1432,7 +1445,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { /* * if a query on a vnode is failed, all retrieve operations from vnode that occurs later - * than this one are actually not necessary, we simply call the tscRetrieveFromVnodeCallBack + * than this one are actually not necessary, we simply call the tscRetrieveFromDnodeCallBack * function to abort current and remain retrieve process. * * NOTE: threadsafe is required. @@ -1453,7 +1466,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; } else { SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL && pNewQueryInfo->pTableMetaInfo[0]->pMetricMeta != NULL); +// assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL && pNewQueryInfo->pTableMetaInfo[0]->pMetricMeta != NULL); tscProcessSql(pNew); return; } @@ -1470,7 +1483,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->subqueryIndex, pState->code); } - tscRetrieveFromVnodeCallBack(param, tres, pState->code); + tscRetrieveFromDnodeCallBack(param, tres, pState->code); } else { // success, proceed to retrieve data from dnode if (vnodeInfo != NULL) { tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, @@ -1481,7 +1494,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->subqueryIndex); } - taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); + taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index aa4e5c93aa7a067c00cc4221e574a34e66c56c1f..ec6881db3f5ae4f1e32dbf790bfb662ae3df84c5 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -53,7 +53,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { const int32_t maxKeySize = TSDB_MAX_TAGS_LEN; // allowed max key size - SCond* cond = tsGetMetricQueryCondPos(pTagCond, uid); + SCond* cond = tsGetSTableQueryCondPos(pTagCond, uid); char join[512] = {0}; if (pTagCond->joinInfo.hasJoin) { @@ -92,7 +92,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { free(tmp); } -SCond* tsGetMetricQueryCondPos(STagCond* pTagCond, uint64_t uid) { +SCond* tsGetSTableQueryCondPos(STagCond* pTagCond, uint64_t uid) { for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { if (uid == pTagCond->cond[i].uid) { return &pTagCond->cond[i]; @@ -102,7 +102,8 @@ SCond* tsGetMetricQueryCondPos(STagCond* pTagCond, uint64_t uid) { return NULL; } -void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) { +// todo refactor by using SArray +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) { size_t len = strlen(str); if (len == 0) { return; @@ -122,7 +123,7 @@ bool tscQueryOnMetric(SSqlCmd* pCmd) { (pCmd->msgType == TSDB_MSG_TYPE_QUERY); } -bool tscQueryMetricTags(SQueryInfo* pQueryInfo) { +bool tscQueryTags(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { if (tscSqlExprGet(pQueryInfo, i)->functionId != TSDB_FUNC_TAGPRJ) { return false; @@ -172,6 +173,7 @@ void tscGetDBInfoFromMeterId(char* tableId, char* db) { } SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx) { +#if 0 if (pMetricmeta == NULL) { tscError("illegal metricmeta"); return 0; @@ -189,6 +191,8 @@ SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx } return (SVnodeSidList*)(pMetricmeta->list[vnodeIdx] + (char*)pMetricmeta); +#endif + } STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) { @@ -221,12 +225,12 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { // for select query super table, the metricmeta can not be null in any cases. if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - assert(pTableMetaInfo->pMetricMeta != NULL); +// assert(pTableMetaInfo->pMetricMeta != NULL); } - if (pTableMetaInfo->pMetricMeta == NULL) { - return false; - } +// if (pTableMetaInfo->pMetricMeta == NULL) { +// return false; +// } if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { return false; @@ -259,12 +263,12 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // only query on tag, not a projection query - if (tscQueryMetricTags(pQueryInfo)) { + if (tscQueryTags(pQueryInfo)) { return false; } // for project query, only the following two function is allowed - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) { @@ -423,9 +427,6 @@ void tscFreeResData(SSqlObj* pSql) { } void tscFreeSqlResult(SSqlObj* pSql) { - //TODO not free - return; - tfree(pSql->res.pRsp); pSql->res.row = 0; pSql->res.numOfRows = 0; @@ -469,8 +470,6 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) { tscFreeSqlCmdData(pCmd); tscTrace("%p free sqlObj partial completed", pSql); - - tscFreeSqlCmdData(pCmd); } void tscFreeSqlObj(SSqlObj* pSql) { @@ -489,10 +488,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pCmd->allocSize = 0; - if (pSql->fp == NULL) { - tsem_destroy(&pSql->rspSem); - tsem_destroy(&pSql->emptyRspSem); - } + tsem_destroy(&pSql->rspSem); free(pSql); } @@ -695,6 +691,49 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, return TSDB_CODE_SUCCESS; } +static void trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { + int32_t firstPartLen = 0; + + STableMeta* pTableMeta = pTableDataBlock->pTableMeta; + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + SSchema* pSchema = tscGetTableSchema(pTableMeta); + + memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); + pDataBlock += sizeof(SSubmitBlk); + + int32_t total = sizeof(int32_t)*2; + for(int32_t i = 0; i < tinfo.numOfColumns; ++i) { + switch (pSchema[i].type) { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_BINARY: { + assert(0); // not support binary yet + firstPartLen += sizeof(int32_t);break; + } + default: + firstPartLen += tDataTypeDesc[pSchema[i].type].nSize; + total += tDataTypeDesc[pSchema[i].type].nSize; + } + } + + char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); + + SSubmitBlk* pBlock = (SSubmitBlk*) pTableDataBlock->pData; + int32_t rows = htons(pBlock->numOfRows); + + for(int32_t i = 0; i < rows; ++i) { + *(int32_t*) pDataBlock = total; + pDataBlock += sizeof(int32_t); + + *(int32_t*) pDataBlock = firstPartLen; + pDataBlock += sizeof(int32_t); + + memcpy(pDataBlock, p, pTableDataBlock->rowSize); + + p += pTableDataBlock->rowSize; + pDataBlock += pTableDataBlock->rowSize; + } +} + int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) { SSqlCmd* pCmd = &pSql->cmd; @@ -716,7 +755,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi return ret; } - int64_t destSize = dataBuf->size + pOneTableBlock->size; + int64_t destSize = dataBuf->size + pOneTableBlock->size + pOneTableBlock->size*sizeof(int32_t)*2; if (dataBuf->nAllocSize < destSize) { while (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = dataBuf->nAllocSize * 1.5; @@ -730,29 +769,33 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi tscError("%p failed to allocate memory for merging submit block, size:%d", pSql, dataBuf->nAllocSize); taosHashCleanup(pVnodeDataBlockHashList); - tfree(dataBuf->pData); tscDestroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); return TSDB_CODE_CLI_OUT_OF_MEMORY; } } - SShellSubmitBlock* pBlocks = (SShellSubmitBlock*)pOneTableBlock->pData; + SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; sortRemoveDuplicates(pOneTableBlock); - char* e = (char*)pBlocks->payLoad + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); + char* e = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); - tscTrace("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, pBlocks->sid, - pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->payLoad), GET_INT64_VAL(e)); + tscTrace("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, + pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(e)); - pBlocks->sid = htonl(pBlocks->sid); + int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + sizeof(int32_t) * 2); + + pBlocks->tid = htonl(pBlocks->tid); pBlocks->uid = htobe64(pBlocks->uid); pBlocks->sversion = htonl(pBlocks->sversion); pBlocks->numOfRows = htons(pBlocks->numOfRows); - - memcpy(dataBuf->pData + dataBuf->size, pOneTableBlock->pData, pOneTableBlock->size); - - dataBuf->size += pOneTableBlock->size; + + pBlocks->len = htonl(len); + + // erase the empty space reserved for binary data + trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock); + dataBuf->size += (len + sizeof(SSubmitBlk)); dataBuf->numOfTables += 1; } @@ -1704,16 +1747,8 @@ bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql) { } int32_t command = pSql->cmd.command; - if (pTscObj->pSql == pSql) { - /* - * in case of taos_connect_a query, the object should all be released, even it is the - * master sql object. Otherwise, the master sql should not be released - */ - if (command == TSDB_SQL_CONNECT && pSql->res.code != TSDB_CODE_SUCCESS) { - return true; - } - - return false; + if (command == TSDB_SQL_CONNECT) { + return true; } if (command == TSDB_SQL_INSERT) { @@ -1894,7 +1929,7 @@ STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST } pTableMetaInfo->pTableMeta = pTableMeta; - pTableMetaInfo->pMetricMeta = pMetricMeta; +// pTableMetaInfo->pMetricMeta = pMetricMeta; pTableMetaInfo->numOfTags = numOfTags; if (tags != NULL) { @@ -1944,7 +1979,7 @@ void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) } taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache); +// taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache); } void tscResetForNextRetrieve(SSqlRes* pRes) { @@ -2083,15 +2118,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); STableMeta* pPrevMeterMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); - SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta); +// SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta); - pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, - pTableMetaInfo->tagColumnIndex); +// pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, +// pTableMetaInfo->tagColumnIndex); } assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1); if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - assert(pFinalInfo->pMetricMeta != NULL); +// assert(pFinalInfo->pMetricMeta != NULL); } tscTrace( @@ -2191,13 +2226,13 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) { +// if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) { return false; - } +// } - int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; - return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vnodeIndex < totalVnode - 1); +// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; +// return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && +// (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vnodeIndex < totalVnode - 1); } void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { @@ -2213,7 +2248,8 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes)); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; + int32_t totalVnode = 0; +// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; while (++pTableMetaInfo->vnodeIndex < totalVnode) { tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1f53d50c1f3b4f57571508060db4f19616eb49de --- /dev/null +++ b/src/client/tests/timeParseTest.cpp @@ -0,0 +1,167 @@ +#include +#include +#include + +#include "taos.h" +#include "tstoken.h" +#include "ttime.h" +#include "tutil.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +extern void deltaToUtcInitOnce(); +/* test parse time function */ +TEST(testCase, parse_time) { + + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + deltaToUtcInitOnce(); + + char t1[] = "2018-1-1 1:1:1.952798"; + char t13[] = "1970-1-1 0:0:0"; + + int64_t time = 0, time1 = 0; + + taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); + + char t2[] = "2018-1-1T1:1:1.952Z"; + taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI); + + EXPECT_EQ(time, 1514739661952 + 28800000); + + char t3[] = "2018-1-1 1:01:01.952"; + taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + char t4[] = "2018-1-1 1:01:01.9"; + char t5[] = "2018-1-1 1:01:1.900"; + char t6[] = "2018-01-01 1:1:1.90"; + char t7[] = "2018-01-01 01:01:01.9"; + char t8[] = "2018-01-01 01:01:01.9007865"; + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t9[] = "2017-4-3 1:1:2.980"; + char t10[] = "2017-4-3T2:1:2.98+9:00"; + taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t11[] = "2017-4-3T2:1:2.98+09:00"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t12[] = "2017-4-3T2:1:2.98+0900"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taos_options(TSDB_OPTION_TIMEZONE, "UTC"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + deltaToUtcInitOnce(); + + char t14[] = "1970-1-1T0:0:0Z"; + taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + char t40[] = "1970-1-1 0:0:0.999999999"; + taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); + + char t41[] = "1997-1-1 0:0:0.999999999"; + taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999); + + int64_t k = timezone; + char t42[] = "1997-1-1T0:0:0.999999999Z"; + taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); + + //////////////////////////////////////////////////////////////////// + // illegal timestamp format + char t15[] = "2017-12-33 0:0:0"; + EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1); + + char t16[] = "2017-12-31 99:0:0"; + EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1); + + char t17[] = "2017-12-31T9:0:0"; + EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1); + + char t18[] = "2017-12-31T9:0:0.Z"; + EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1); + + char t19[] = "2017-12-31 9:0:0.-1"; + EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1); + + char t20[] = "2017-12-31 9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0); + EXPECT_EQ(time, 1514682000100); + + char t21[] = "2017-12-31T9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1); + + char t22[] = "2017-12-31 9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0); + + char t23[] = "2017-12-31T9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0); + + + //======================== add some case ============================// + + char b1[] = "9999-12-31 23:59:59.999"; + taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 253402271999999); + + + char b2[] = "2020-01-01 01:01:01.321"; + taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1577811661321); + + taos_options(TSDB_OPTION_TIMEZONE, "America/New_York"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND); +} + + diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index aff239712bf1e19ae9d4933d956790e7ed2bec2c..4e8afd4f0eec26b42842fd4b4ddfcff05d876a13 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -81,11 +81,13 @@ STSchema *tdDecodeSchema(void **psrc); */ typedef void *SDataRow; + #define TD_DATA_ROW_HEAD_SIZE (2 * sizeof(int32_t)) #define dataRowLen(r) (*(int32_t *)(r)) #define dataRowFLen(r) (*(int32_t *)((char *)(r) + sizeof(int32_t))) #define dataRowTuple(r) ((char *)(r) + TD_DATA_ROW_HEAD_SIZE) +#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r))) #define dataRowSetLen(r, l) (dataRowLen(r) = (l)) #define dataRowSetFLen(r, l) (dataRowFLen(r) = (l)) #define dataRowIdx(r, i) ((char *)(r) + i) @@ -103,11 +105,36 @@ SDataRow tdDataRowDup(SDataRow row); // ----------------- Data column structure typedef struct SDataCol { - int64_t len; - char data[]; + int8_t type; + int16_t colId; + int bytes; + int len; + int offset; + void * pData; } SDataCol; -void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter); +typedef struct { + int maxRowSize; + int maxCols; // max number of columns + int maxPoints; // max number of points + int numOfPoints; + int numOfCols; // Total number of cols + int sversion; // TODO: set sversion + void * buf; + SDataCol cols[]; +} SDataCols; + +#define keyCol(pCols) (&((pCols)->cols[0])) // Key column +#define dataColsKeyAt(pCols, idx) ((int64_t *)(keyCol(pCols)->pData))[(idx)] +#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0) +#define dataColsKeyLast(pCols) dataColsKeyAt(pCols, (pCols)->numOfPoints - 1) + +SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); +void tdResetDataCols(SDataCols *pCols); +void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); +void tdFreeDataCols(SDataCols *pCols); +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols); +void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); #ifdef __cplusplus } diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 9c356b0cbc71671ee8a7d917bf18c0b988f0cb1f..0496fc6feb313597b343f10e366e838a46f1697f 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -150,7 +150,7 @@ void tdFreeSchema(STSchema *pSchema) { */ void tdUpdateSchema(STSchema *pSchema) { STColumn *pCol = NULL; - int32_t offset = 0; + int32_t offset = TD_DATA_ROW_HEAD_SIZE; for (int i = 0; i < schemaNCols(pSchema); i++) { pCol = schemaColAt(pSchema, i); colSetOffset(pCol, offset); @@ -294,14 +294,79 @@ SDataRow tdDataRowDup(SDataRow row) { return trow; } -void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter) { - int row = *iter; +SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { + SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * maxCols); + if (pCols == NULL) return NULL; + pCols->maxRowSize = maxRowSize; + pCols->maxCols = maxCols; + pCols->maxPoints = maxRows; + + pCols->buf = malloc(maxRowSize * maxRows); + if (pCols->buf == NULL) { + free(pCols); + return NULL; + } + + return pCols; +} + +void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { + // assert(schemaNCols(pSchema) <= pCols->numOfCols); + tdResetDataCols(pCols); + pCols->numOfCols = schemaNCols(pSchema); + + pCols->cols[0].pData = pCols->buf; for (int i = 0; i < schemaNCols(pSchema); i++) { - // TODO + if (i > 0) { + pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints; + } + pCols->cols[i].type = colType(schemaColAt(pSchema, i)); + pCols->cols[i].bytes = colBytes(schemaColAt(pSchema, i)); + pCols->cols[i].offset = colOffset(schemaColAt(pSchema, i)); + pCols->cols[i].colId = colColId(schemaColAt(pSchema, i)); } - *iter = row + 1; + return pCols; +} + +void tdFreeDataCols(SDataCols *pCols) { + if (pCols) { + if (pCols->buf) free(pCols->buf); + free(pCols); + } +} + +void tdResetDataCols(SDataCols *pCols) { + pCols->numOfPoints = 0; + for (int i = 0; i < pCols->maxCols; i++) { + pCols->cols[i].len = 0; + } +} + +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) { + TSKEY key = dataRowKey(row); + for (int i = 0; i < pCols->numOfCols; i++) { + SDataCol *pCol = pCols->cols + i; + memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, pCol->offset), pCol->bytes); + pCol->len += pCol->bytes; + } + pCols->numOfPoints++; +} +// Pop pointsToPop points from the SDataCols +void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) { + int pointsLeft = pCols->numOfPoints - pointsToPop; + + for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { + SDataCol *p_col = pCols->cols + iCol; + if (p_col->len > 0) { + p_col->len = TYPE_BYTES[p_col->type] * pointsLeft; + if (pointsLeft > 0) { + memmove((void *)(p_col->pData), (void *)((char *)(p_col->pData) + TYPE_BYTES[p_col->type] * pointsToPop), p_col->len); + } + } + } + pCols->numOfPoints = pointsLeft; } /** diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 298eb6110c8dab7ac34a24446d851b37a6b5dace..d85782b463f4ca579c9874ee1050593ff01f40ca 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode sdb taos_static monitor http tsdb) + TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb) #IF (TD_CLUSTER) # TARGET_LINK_LIBRARIES(taosd dcluster) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index f6976aa2cc5f2b40471bb013a6241db4317651d8..4d335d73534b452b71753b63d6d1b814b0df674b 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -225,17 +225,17 @@ static int32_t dnodeOpenVnode(int32_t vnode, char *rootDir) { vnodeObj.status = TSDB_VN_STATUS_NOT_READY; vnodeObj.refCount = 1; vnodeObj.version = 0; - vnodeObj.wworker = dnodeAllocateWriteWorker(&vnodeObj); - vnodeObj.rworker = dnodeAllocateReadWorker(&vnodeObj); vnodeObj.wal = NULL; vnodeObj.tsdb = pTsdb; vnodeObj.replica = NULL; vnodeObj.events = NULL; vnodeObj.cq = NULL; - taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *) (&vnodeObj)); + SVnodeObj *pVnode = (SVnodeObj *)taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *)(&vnodeObj)); + pVnode->wworker = dnodeAllocateWriteWorker(pVnode); + pVnode->rworker = dnodeAllocateReadWorker(pVnode); - dTrace("open vnode:%d in %s", vnodeObj.vgId, rootDir); + dTrace("open vnode:%d in %s", pVnode->vgId, rootDir); return TSDB_CODE_SUCCESS; } @@ -314,17 +314,17 @@ static int32_t dnodeCreateVnode(SMDCreateVnodeMsg *pVnodeCfg) { vnodeObj.status = TSDB_VN_STATUS_NOT_READY; vnodeObj.refCount = 1; vnodeObj.version = 0; - vnodeObj.wworker = dnodeAllocateWriteWorker(&vnodeObj); - vnodeObj.rworker = dnodeAllocateReadWorker(&vnodeObj); vnodeObj.wal = NULL; vnodeObj.tsdb = pTsdb; vnodeObj.replica = NULL; vnodeObj.events = NULL; vnodeObj.cq = NULL; - taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *) (&vnodeObj)); + SVnodeObj *pVnode = (SVnodeObj *)taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *)(&vnodeObj)); + pVnode->wworker = dnodeAllocateWriteWorker(pVnode); + pVnode->rworker = dnodeAllocateReadWorker(pVnode); - dPrint("vgroup:%d, vnode:%d is created", vnodeObj.vgId, vnodeObj.vgId); + dPrint("vgroup:%d, vnode:%d is created", pVnode->vgId, pVnode->vgId); return TSDB_CODE_SUCCESS; } diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index a870a9e1597fce79e8271e5834672cf808431057..cb25cbfd525623d3832187135beadb3981475af9 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -76,7 +76,9 @@ void dnodeRead(SRpcMsg *pMsg) { int32_t leftLen = pMsg->contLen; char *pCont = (char *) pMsg->pCont; SRpcContext *pRpcContext = NULL; - + + dTrace("dnode read msg disposal"); + // SMsgDesc *pDesc = pCont; // pDesc->numOfVnodes = htonl(pDesc->numOfVnodes); // pCont += sizeof(SMsgDesc); @@ -90,8 +92,8 @@ void dnodeRead(SRpcMsg *pMsg) { while (leftLen > 0) { SMsgHead *pHead = (SMsgHead *) pCont; - pHead->vgId = 1; //htonl(pHead->vgId); - pHead->contLen = pMsg->contLen; //htonl(pHead->contLen); + pHead->vgId = htonl(pHead->vgId); + pHead->contLen = htonl(pHead->contLen); void *pVnode = dnodeGetVnode(pHead->vgId); if (pVnode == NULL) { @@ -162,7 +164,7 @@ static void *dnodeProcessReadQueue(void *param) { void *pVnode; while (1) { - if (taosReadQitemFromQset(qset, &type, &pReadMsg, &pVnode) == 0) { + if (taosReadQitemFromQset(qset, &type, (void **)&pReadMsg, (void **)&pVnode) == 0) { dnodeHandleIdleReadWorker(); continue; } @@ -174,10 +176,10 @@ static void *dnodeProcessReadQueue(void *param) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; } - dnodeProcessReadResult(pVnode, pReadMsg); +// dnodeProcessReadResult(pVnode, pReadMsg); taosFreeQitem(pReadMsg); - dnodeReleaseVnode(pVnode); + dnodeReleaseVnode(pVnode); } return NULL; @@ -218,88 +220,92 @@ static void dnodeProcessReadResult(void *pVnode, SReadMsg *pRead) { code = terrno; } - SRpcMsg rsp; - rsp.handle = pRead->rpcMsg.handle; - rsp.code = code; - rsp.pCont = NULL; - rpcSendResponse(&rsp); + //TODO: query handle is returned by dnodeProcessQueryMsg + if (0) { + SRpcMsg rsp; + rsp.handle = pRead->rpcMsg.handle; + rsp.code = code; + rsp.pCont = NULL; + rpcSendResponse(&rsp); + } + rpcFreeCont(pRead->rpcMsg.pCont); // free the received message } -static void dnodeProcessQueryMsg(SReadMsg *pMsg) { +static void dnodeContinueExecuteQuery(void* pVnode, void* qhandle, SReadMsg *pMsg) { + + SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); + pRead->rpcMsg = pMsg->rpcMsg; + pRead->pCont = qhandle; + pRead->contLen = 0; + pRead->pRpcContext = pMsg->pRpcContext; + pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; + + taos_queue queue = dnodeGetVnodeRworker(pVnode); + taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); +} + +static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) { SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pMsg->pCont; SQInfo* pQInfo = NULL; - int32_t code = qCreateQueryInfo(pQueryTableMsg, &pQInfo); + if (pMsg->contLen != 0) { + void* tsdb = dnodeGetVnodeTsdb(pVnode); + int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo); - SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); - pRsp->code = code; - pRsp->qhandle = htobe64((uint64_t) (pQInfo)); - - SRpcMsg rpcRsp = { - .handle = pMsg->rpcMsg.handle, - .pCont = pRsp, - .contLen = sizeof(SQueryTableRsp), - .code = code, - .msgType = 0 - }; + SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); + pRsp->code = code; + pRsp->qhandle = htobe64((uint64_t) (pQInfo)); - // do execute query - qTableQuery(pQInfo); + SRpcMsg rpcRsp = { + .handle = pMsg->rpcMsg.handle, + .pCont = pRsp, + .contLen = sizeof(SQueryTableRsp), + .code = code, + .msgType = 0 + }; - rpcSendResponse(&rpcRsp); + rpcSendResponse(&rpcRsp); + } else { + pQInfo = pMsg->pCont; + } + + qTableQuery(pQInfo); // do execute query } -static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { +static void dnodeProcessRetrieveMsg(void *pVnode, SReadMsg *pMsg) { SRetrieveTableMsg *pRetrieve = pMsg->pCont; - void *pQInfo = htobe64(pRetrieve->qhandle); + void *pQInfo = (void*) htobe64(pRetrieve->qhandle); - dTrace("retrieve msg is disposed, qInfo:%p", pQInfo); - - int32_t rowSize = 0; - int32_t numOfRows = 0; + dTrace("QInfo:%p vgId:%d, retrieve msg is received", pQInfo, pRetrieve->header.vgId); int32_t contLen = 0; - SRpcMsg rpcRsp = {0}; + SRetrieveTableRsp *pRsp = NULL; - int32_t code = qRetrieveQueryResultInfo(pQInfo, &numOfRows, &rowSize); + int32_t code = qRetrieveQueryResultInfo(pQInfo); if (code != TSDB_CODE_SUCCESS) { contLen = sizeof(SRetrieveTableRsp); - - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); - pRsp->numOfRows = 0; - pRsp->precision = 0; - pRsp->offset = 0; - pRsp->useconds = 0; - - rpcRsp = (SRpcMsg) { - .handle = pMsg->rpcMsg.handle, - .pCont = pRsp, - .contLen = contLen, - .code = code, - .msgType = 0 - }; - - //todo free qinfo - } else { - contLen = 100; - - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); - pRsp->numOfRows = 0; - pRsp->precision = 0; - pRsp->offset = 0; - pRsp->useconds = 0; - *(int64_t*) pRsp->data = 1000; + pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); + memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + } else { + // todo check code and handle error in build result set + code = qDumpRetrieveResult(pQInfo, &pRsp, &contLen); - rpcRsp = (SRpcMsg) { - .handle = pMsg->rpcMsg.handle, - .pCont = pRsp, - .contLen = contLen, - .code = code, - .msgType = 0 - }; + if (qHasMoreResultsToRetrieve(pQInfo)) { + dnodeContinueExecuteQuery(pVnode, pQInfo, pMsg); + } else { // no further execution invoked, release the ref to vnode + dnodeProcessReadResult(pVnode, pMsg); + } } + SRpcMsg rpcRsp = (SRpcMsg) { + .handle = pMsg->rpcMsg.handle, + .pCont = pRsp, + .contLen = contLen, + .code = code, + .msgType = 0 + }; + rpcSendResponse(&rpcRsp); } diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index 5dd075e4df4be60f4dac199cbc1d3ba860114312..7ec206f366a3e1b7151a6dae043dbc7937eb2a35 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -53,12 +53,12 @@ typedef struct _thread_obj { static void (*dnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(void *, SWriteMsg *); static void *dnodeProcessWriteQueue(void *param); static void dnodeHandleIdleWorker(SWriteWorker *pWorker); -static void dnodeProcessWriteResult(SWriteMsg *pWrite); -static void dnodeProcessSubmitMsg(SWriteMsg *pMsg); -static void dnodeProcessCreateTableMsg(SWriteMsg *pMsg); -static void dnodeProcessDropTableMsg(SWriteMsg *pMsg); -static void dnodeProcessAlterTableMsg(SWriteMsg *pMsg); -static void dnodeProcessDropStableMsg(SWriteMsg *pMsg); +static void dnodeProcessWriteResult(void *pVnode, SWriteMsg *pWrite); +static void dnodeProcessSubmitMsg(void *pVnode, SWriteMsg *pMsg); +static void dnodeProcessCreateTableMsg(void *pVnode, SWriteMsg *pMsg); +static void dnodeProcessDropTableMsg(void *pVnode, SWriteMsg *pMsg); +static void dnodeProcessAlterTableMsg(void *pVnode, SWriteMsg *pMsg); +static void dnodeProcessDropStableMsg(void *pVnode, SWriteMsg *pMsg); SWriteWorkerPool wWorkerPool; @@ -193,20 +193,20 @@ static void *dnodeProcessWriteQueue(void *param) { continue; } - for (int32_t i=0; iwhandle, writeMsg.rpcMsg.msgType, writeMsg.pCont, writeMsg.contLen); } - + // flush WAL file // walFsync(pVnode->whandle); // browse all items, and process them one by one taosResetQitems(qall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, &type, &pWriteMsg); + taosGetQitem(qall, &type, (void **)&pWriteMsg); terrno = 0; if (dnodeProcessWriteMsgFp[pWriteMsg->rpcMsg.msgType]) { @@ -218,7 +218,6 @@ static void *dnodeProcessWriteQueue(void *param) { dnodeProcessWriteResult(pVnode, pWriteMsg); taosFreeQitem(pWriteMsg); } - } taosFreeQall(qall); @@ -270,7 +269,7 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { } } -static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) { +static void dnodeProcessSubmitMsg(void *pVnode, SWriteMsg *pMsg) { dTrace("submit msg is disposed"); SShellSubmitRspMsg *pRsp = rpcMallocCont(sizeof(SShellSubmitRspMsg)); @@ -279,7 +278,10 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) { pRsp->affectedRows = htonl(1); pRsp->numOfFailedBlocks = 0; - // todo write to tsdb + void* tsdb = dnodeGetVnodeTsdb(pVnode); + assert(tsdb != NULL); + + tsdbInsertData(tsdb, pMsg->pCont); SRpcMsg rpcRsp = { .handle = pMsg->rpcMsg.handle, @@ -288,10 +290,11 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) { .code = 0, .msgType = 0 }; + rpcSendResponse(&rpcRsp); } -static void dnodeProcessCreateTableMsg(SWriteMsg *pMsg) { +static void dnodeProcessCreateTableMsg(void *pVnode, SWriteMsg *pMsg) { SMDCreateTableMsg *pTable = pMsg->rpcMsg.pCont; SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; @@ -340,16 +343,16 @@ static void dnodeProcessCreateTableMsg(SWriteMsg *pMsg) { tsdbTableSetTagValue(&tCfg, dataRow, false); } - void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + void *pTsdb = dnodeGetVnodeTsdb(pVnode); rpcRsp.code = tsdbCreateTable(pTsdb, &tCfg); - dnodeReleaseVnode(pMsg->pVnode); + dnodeReleaseVnode(pVnode); dTrace("table:%s, create table result:%s", pTable->tableId, tstrerror(rpcRsp.code)); rpcSendResponse(&rpcRsp); } -static void dnodeProcessDropTableMsg(SWriteMsg *pMsg) { +static void dnodeProcessDropTableMsg(void *pVnode, SWriteMsg *pMsg) { SMDDropTableMsg *pTable = pMsg->rpcMsg.pCont; SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; @@ -359,16 +362,16 @@ static void dnodeProcessDropTableMsg(SWriteMsg *pMsg) { .tid = htonl(pTable->sid) }; - void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + void *pTsdb = dnodeGetVnodeTsdb(pVnode); rpcRsp.code = tsdbDropTable(pTsdb, tableId); - dnodeReleaseVnode(pMsg->pVnode); + dnodeReleaseVnode(pVnode); dTrace("table:%s, drop table result:%s", pTable->tableId, tstrerror(rpcRsp.code)); rpcSendResponse(&rpcRsp); } -static void dnodeProcessAlterTableMsg(SWriteMsg *pMsg) { +static void dnodeProcessAlterTableMsg(void *pVnode, SWriteMsg *pMsg) { SMDCreateTableMsg *pTable = pMsg->rpcMsg.pCont; SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; @@ -417,16 +420,16 @@ static void dnodeProcessAlterTableMsg(SWriteMsg *pMsg) { tsdbTableSetTagValue(&tCfg, dataRow, false); } - void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + void *pTsdb = dnodeGetVnodeTsdb(pVnode); rpcRsp.code = tsdbAlterTable(pTsdb, &tCfg); - dnodeReleaseVnode(pMsg->pVnode); + dnodeReleaseVnode(pVnode); dTrace("table:%s, alter table result:%s", pTable->tableId, tstrerror(rpcRsp.code)); rpcSendResponse(&rpcRsp); } -static void dnodeProcessDropStableMsg(SWriteMsg *pMsg) { +static void dnodeProcessDropStableMsg(void *pVnode, SWriteMsg *pMsg) { SMDDropSTableMsg *pTable = pMsg->rpcMsg.pCont; SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; @@ -438,7 +441,7 @@ static void dnodeProcessDropStableMsg(SWriteMsg *pMsg) { //rpcRsp.code = tsdbDropSTable(pTsdb, pTable->uid); rpcRsp.code = TSDB_CODE_SUCCESS; - dnodeReleaseVnode(pMsg->pVnode); + dnodeReleaseVnode(pVnode); dTrace("stable:%s, drop stable result:%s", pTable->tableId, tstrerror(rpcRsp.code)); rpcSendResponse(&rpcRsp); diff --git a/src/inc/mnode.h b/src/inc/mnode.h index be152fbf3cbc6000cb40affbe99ea07e3c62b05e..e6cc0f1367526599b5d5e01b26675f9d60278114 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -25,7 +25,6 @@ extern "C" { #include "taosdef.h" #include "taosmsg.h" #include "taoserror.h" -#include "sdb.h" #include "tglobalcfg.h" #include "thash.h" #include "tidpool.h" @@ -54,8 +53,8 @@ typedef struct { int8_t numOfMnodes; int32_t numOfDnodes; char mnodeName[TSDB_DNODE_NAME_LEN + 1]; - char reserved[7]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; int syncFd; void *hbTimer; void *pSync; @@ -79,8 +78,8 @@ typedef struct { float lbScore; // calc in balance function int32_t customScore; // config by user char dnodeName[TSDB_DNODE_NAME_LEN + 1]; - char reserved[7]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; SVnodeLoad vload[TSDB_MAX_VNODES]; int32_t status; uint32_t lastReboot; // time stamp for last reboot @@ -99,66 +98,46 @@ typedef struct { } SVnodeGid; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - int8_t type; - int8_t dirty; - uint64_t uid; - int32_t sid; - int32_t vgId; - int64_t createdTime; + char tableId[TSDB_TABLE_ID_LEN]; + int8_t type; + int8_t dirty; } STableInfo; struct _vg_obj; typedef struct SSuperTableObj { - char tableId[TSDB_TABLE_ID_LEN + 1]; - int8_t type; - int8_t dirty; - uint64_t uid; - int32_t sid; - int32_t vgId; - int64_t createdTime; - int32_t sversion; - int32_t numOfColumns; - int32_t numOfTags; - int8_t reserved[5]; - int8_t updateEnd[1]; - int32_t numOfTables; - int16_t nextColId; - SSchema *schema; + STableInfo info; + uint64_t uid; + int64_t createdTime; + int32_t sversion; + int32_t numOfColumns; + int32_t numOfTags; + int8_t reserved[15]; + int8_t updateEnd[1]; + int32_t numOfTables; + int16_t nextColId; + SSchema * schema; } SSuperTableObj; typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; - int8_t type; - int8_t dirty; - uint64_t uid; - int32_t sid; - int32_t vgId; - int64_t createdTime; - char superTableId[TSDB_TABLE_ID_LEN + 1]; - int8_t reserved[1]; - int8_t updateEnd[1]; + STableInfo info; + uint64_t uid; + int64_t createdTime; + int32_t sversion; //used by normal table + int32_t numOfColumns; //used by normal table + int32_t sid; + int32_t vgId; + char superTableId[TSDB_TABLE_ID_LEN + 1]; + int32_t sqlLen; + int8_t reserved[1]; + int8_t updateEnd[1]; + int16_t nextColId; //used by normal table + char* sql; //used by normal table + SSchema* schema; //used by normal table SSuperTableObj *superTable; } SChildTableObj; -typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - int8_t type; - int8_t dirty; - uint64_t uid; - int32_t sid; - int32_t vgId; - int64_t createdTime; - int32_t sversion; - int32_t numOfColumns; - int32_t sqlLen; - int8_t reserved[3]; - int8_t updateEnd[1]; - char* sql; //null-terminated string - int16_t nextColId; - SSchema* schema; -} SNormalTableObj; +struct _db_obj; typedef struct _vg_obj { uint32_t vgId; @@ -166,15 +145,16 @@ typedef struct _vg_obj { int64_t createdTime; SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT]; int32_t numOfVnodes; - int32_t numOfTables; int32_t lbIp; int32_t lbTime; int8_t lbStatus; int8_t reserved[14]; int8_t updateEnd[1]; struct _vg_obj *prev, *next; + struct _db_obj *pDb; + int32_t numOfTables; void * idPool; - STableInfo ** tableList; + SChildTableObj ** tableList; } SVgObj; typedef struct _db_obj { @@ -182,8 +162,8 @@ typedef struct _db_obj { int8_t dirty; int64_t createdTime; SDbCfg cfg; - char reserved[15]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; struct _db_obj *prev, *next; int32_t numOfVgroups; int32_t numOfTables; @@ -201,7 +181,7 @@ typedef struct _user_obj { int64_t createdTime; int8_t superAuth; int8_t writeAuth; - int8_t reserved[16]; + int8_t reserved[13]; int8_t updateEnd[1]; struct _user_obj *prev, *next; struct _acctObj * pAcct; @@ -258,9 +238,10 @@ typedef struct { typedef struct { uint8_t msgType; - int8_t expected; + int8_t usePublicIp; int8_t received; int8_t successed; + int8_t expected; int32_t contLen; int32_t code; void *ahandle; diff --git a/src/inc/sdb.h b/src/inc/sdb.h deleted file mode 100644 index 4b4de1ac4b9b2575493f289f98660dbbb6704911..0000000000000000000000000000000000000000 --- a/src/inc/sdb.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SDB_H -#define TDENGINE_SDB_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "taosmsg.h" -#include "taosdef.h" - -extern uint16_t tsMgmtMgmtPort; -extern uint16_t tsMgmtSyncPort; -extern int tsMgmtPeerHBTimer; // seconds -extern char * sdbStatusStr[]; -extern char * sdbRoleStr[]; -extern int sdbMaster; -extern SRpcIpSet *pSdbIpList; -extern SRpcIpSet *pSdbPublicIpList; - -extern void (*sdbWorkAsMasterCallback)(); // this function pointer will be set by taosd - -enum _keytype { - SDB_KEYTYPE_STRING, SDB_KEYTYPE_UINT32, SDB_KEYTYPE_AUTO, SDB_KEYTYPE_RECYCLE, SDB_KEYTYPE_MAX -}; - -#define SDB_ROLE_UNAPPROVED 0 -#define SDB_ROLE_UNDECIDED 1 -#define SDB_ROLE_MASTER 2 -#define SDB_ROLE_SLAVE 3 - -#define SDB_STATUS_OFFLINE 0 -#define SDB_STATUS_UNSYNCED 1 -#define SDB_STATUS_SYNCING 2 -#define SDB_STATUS_SERVING 3 -#define SDB_STATUS_DELETED 4 - -enum _sdbaction { - SDB_TYPE_INSERT, - SDB_TYPE_DELETE, - SDB_TYPE_UPDATE, - SDB_TYPE_DECODE, - SDB_TYPE_ENCODE, - SDB_TYPE_BEFORE_BATCH_UPDATE, - SDB_TYPE_BATCH_UPDATE, - SDB_TYPE_AFTER_BATCH_UPDATE, - SDB_TYPE_RESET, - SDB_TYPE_DESTROY, - SDB_MAX_ACTION_TYPES -}; - -#define SDB_MAX_PEERS 4 -typedef struct { - uint32_t ip; - uint32_t publicIp; - char ipstr[20]; - char zone[12]; - char role; - int64_t createdTime; - uint64_t dbVersion; - int64_t lostTime; - char status; - char numOfMnodes; - int numOfDnodes; - char updateEnd[1]; - - // internal - int syncFd; - void *hbTimer; - void *pSync; -} SSdbPeer; - -extern SSdbPeer *sdbPeer[]; -#define sdbInited (sdbPeer[0]) -#define sdbStatus (sdbPeer[0]->status) - -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int, int *)); - -void *sdbGetRow(void *handle, void *key); - -int64_t sdbInsertRow(void *handle, void *row, int rowSize); - -int sdbDeleteRow(void *handle, void *key); - -int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated); - -void *sdbFetchRow(void *handle, void *pNode, void **ppRow); - -int sdbBatchUpdateRow(void *handle, void *row, int rowSize); - -int64_t sdbGetId(void *handle); - -int64_t sdbGetNumOfRows(void *handle); - -void sdbSaveSnapShot(void *handle); - -void sdbCloseTable(void *handle); - -int sdbRemovePeerByIp(uint32_t ip); - -int sdbInitPeers(char *directory); - -void sdbCleanUpPeers(); - -int64_t sdbGetVersion(); - -int32_t sdbGetRunStatus(); - -#define TSDB_MAX_NORMAL_TABLES 10000 -#define TSDB_MAX_SUPER_TABLES 1000 - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_SDB_H diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 5da8aa191b5e1ce5072a80599a0c64b3f44d9f15..d6a9447e3d192120a2712b46d1b2310c2732344f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -176,7 +176,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_COLUMNS 256 #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns -#define TSDB_DNODE_NAME_LEN 63 +#define TSDB_DNODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 192 #define TSDB_DB_NAME_LEN 32 #define TSDB_COL_NAME_LEN 64 @@ -309,6 +309,16 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_SESSIONS_PER_VNODE (300) #define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES) +#define TSDB_MAX_MNODES 5 +#define TSDB_MAX_DNODES 10 +#define TSDB_MAX_ACCOUNTS 10 +#define TSDB_MAX_USERS 20 +#define TSDB_MAX_DBS 100 +#define TSDB_MAX_VGROUPS 1000 +#define TSDB_MAX_SUPER_TABLES 100 +#define TSDB_MAX_NORMAL_TABLES 1000 +#define TSDB_MAX_CHILD_TABLES 100000 + enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 467c2a2995ac15407364ed0413aa3979e4a43676..4632c67c3cd5dc65267a151f63dcad40cea24b1f 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -20,122 +20,122 @@ extern "C" { #endif -#include #include +#include #include "taosdef.h" #include "taoserror.h" #include "trpc.h" // message type -#define TSDB_MSG_TYPE_REG 1 -#define TSDB_MSG_TYPE_REG_RSP 2 -#define TSDB_MSG_TYPE_SUBMIT 3 -#define TSDB_MSG_TYPE_SUBMIT_RSP 4 -#define TSDB_MSG_TYPE_QUERY 5 -#define TSDB_MSG_TYPE_QUERY_RSP 6 -#define TSDB_MSG_TYPE_RETRIEVE 7 -#define TSDB_MSG_TYPE_RETRIEVE_RSP 8 +#define TSDB_MSG_TYPE_REG 1 +#define TSDB_MSG_TYPE_REG_RSP 2 +#define TSDB_MSG_TYPE_SUBMIT 3 +#define TSDB_MSG_TYPE_SUBMIT_RSP 4 +#define TSDB_MSG_TYPE_QUERY 5 +#define TSDB_MSG_TYPE_QUERY_RSP 6 +#define TSDB_MSG_TYPE_RETRIEVE 7 +#define TSDB_MSG_TYPE_RETRIEVE_RSP 8 // message from mnode to dnode -#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9 -#define TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP 10 -#define TSDB_MSG_TYPE_MD_DROP_TABLE 11 -#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12 -#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13 -#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14 -#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15 -#define TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP 16 -#define TSDB_MSG_TYPE_MD_DROP_VNODE 17 -#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18 -#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19 -#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20 -#define TSDB_MSG_TYPE_MD_DROP_STABLE 21 -#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22 -#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23 -#define TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP 24 -#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25 -#define TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP 26 +#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9 +#define TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP 10 +#define TSDB_MSG_TYPE_MD_DROP_TABLE 11 +#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12 +#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13 +#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14 +#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15 +#define TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP 16 +#define TSDB_MSG_TYPE_MD_DROP_VNODE 17 +#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18 +#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19 +#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20 +#define TSDB_MSG_TYPE_MD_DROP_STABLE 21 +#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22 +#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23 +#define TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP 24 +#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25 +#define TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP 26 // message from client to mnode -#define TSDB_MSG_TYPE_CM_CONNECT 31 -#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32 -#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33 -#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34 -#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35 -#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36 -#define TSDB_MSG_TYPE_CM_DROP_ACCT 37 -#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38 -#define TSDB_MSG_TYPE_CM_CREATE_USER 39 -#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40 -#define TSDB_MSG_TYPE_CM_ALTER_USER 41 -#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42 -#define TSDB_MSG_TYPE_CM_DROP_USER 43 -#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44 -#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45 -#define TSDB_MSG_TYPE_CM_CREATE_DNODE_RSP 46 -#define TSDB_MSG_TYPE_CM_DROP_DNODE 47 -#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48 -#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE -#define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP -#define TSDB_MSG_TYPE_CM_CREATE_DB 49 -#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50 -#define TSDB_MSG_TYPE_CM_DROP_DB 51 -#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52 -#define TSDB_MSG_TYPE_CM_USE_DB 53 -#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54 -#define TSDB_MSG_TYPE_CM_ALTER_DB 55 -#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56 -#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57 -#define TSDB_MSG_TYPE_CM_CREATE_TABLE_RSP 58 -#define TSDB_MSG_TYPE_CM_DROP_TABLE 59 -#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60 -#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61 -#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 -#define TSDB_MSG_TYPE_CM_TABLE_META 63 -#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 -#define TSDB_MSG_TYPE_CM_STABLE_META 65 -#define TSDB_MSG_TYPE_CM_STABLE_META_RSP 66 -#define TSDB_MSG_TYPE_CM_TABLES_META 67 -#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 -#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 -#define TSDB_MSG_TYPE_CM_ALTER_STREAM_RSP 70 -#define TSDB_MSG_TYPE_CM_SHOW 71 -#define TSDB_MSG_TYPE_CM_SHOW_RSP 72 -#define TSDB_MSG_TYPE_CM_KILL_QUERY 73 -#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74 -#define TSDB_MSG_TYPE_CM_KILL_STREAM 75 -#define TSDB_MSG_TYPE_CM_KILL_STREAM_RSP 76 -#define TSDB_MSG_TYPE_CM_KILL_CONN 77 -#define TSDB_MSG_TYPE_CM_KILL_CONN_RSP 78 -#define TSDB_MSG_TYPE_CM_HEARTBEAT 79 -#define TSDB_MSG_TYPE_CM_HEARTBEAT_RSP 80 +#define TSDB_MSG_TYPE_CM_CONNECT 31 +#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32 +#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33 +#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34 +#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35 +#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36 +#define TSDB_MSG_TYPE_CM_DROP_ACCT 37 +#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38 +#define TSDB_MSG_TYPE_CM_CREATE_USER 39 +#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40 +#define TSDB_MSG_TYPE_CM_ALTER_USER 41 +#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42 +#define TSDB_MSG_TYPE_CM_DROP_USER 43 +#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44 +#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45 +#define TSDB_MSG_TYPE_CM_CREATE_DNODE_RSP 46 +#define TSDB_MSG_TYPE_CM_DROP_DNODE 47 +#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48 +#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE +#define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP +#define TSDB_MSG_TYPE_CM_CREATE_DB 49 +#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50 +#define TSDB_MSG_TYPE_CM_DROP_DB 51 +#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52 +#define TSDB_MSG_TYPE_CM_USE_DB 53 +#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54 +#define TSDB_MSG_TYPE_CM_ALTER_DB 55 +#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56 +#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57 +#define TSDB_MSG_TYPE_CM_CREATE_TABLE_RSP 58 +#define TSDB_MSG_TYPE_CM_DROP_TABLE 59 +#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60 +#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61 +#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 +#define TSDB_MSG_TYPE_CM_TABLE_META 63 +#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 +#define TSDB_MSG_TYPE_CM_STABLE_VGROUP 65 +#define TSDB_MSG_TYPE_CM_STABLE_VGROUP_RSP 66 +#define TSDB_MSG_TYPE_CM_TABLES_META 67 +#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 +#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 +#define TSDB_MSG_TYPE_CM_ALTER_STREAM_RSP 70 +#define TSDB_MSG_TYPE_CM_SHOW 71 +#define TSDB_MSG_TYPE_CM_SHOW_RSP 72 +#define TSDB_MSG_TYPE_CM_KILL_QUERY 73 +#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74 +#define TSDB_MSG_TYPE_CM_KILL_STREAM 75 +#define TSDB_MSG_TYPE_CM_KILL_STREAM_RSP 76 +#define TSDB_MSG_TYPE_CM_KILL_CONN 77 +#define TSDB_MSG_TYPE_CM_KILL_CONN_RSP 78 +#define TSDB_MSG_TYPE_CM_HEARTBEAT 79 +#define TSDB_MSG_TYPE_CM_HEARTBEAT_RSP 80 // message from dnode to mnode -#define TSDB_MSG_TYPE_DM_CONFIG_TABLE 91 -#define TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP 92 -#define TSDB_MSG_TYPE_DM_CONFIG_VNODE 93 -#define TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP 94 -#define TSDB_MSG_TYPE_DM_STATUS 95 -#define TSDB_MSG_TYPE_DM_STATUS_RSP 96 -#define TSDB_MSG_TYPE_DM_GRANT 97 -#define TSDB_MSG_TYPE_DM_GRANT_RSP 98 - -#define TSDB_MSG_TYPE_SDB_SYNC 101 -#define TSDB_MSG_TYPE_SDB_SYNC_RSP 102 -#define TSDB_MSG_TYPE_SDB_FORWARD 103 -#define TSDB_MSG_TYPE_SDB_FORWARD_RSP 104 - -#define TSDB_MSG_TYPE_MAX 105 +#define TSDB_MSG_TYPE_DM_CONFIG_TABLE 91 +#define TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP 92 +#define TSDB_MSG_TYPE_DM_CONFIG_VNODE 93 +#define TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP 94 +#define TSDB_MSG_TYPE_DM_STATUS 95 +#define TSDB_MSG_TYPE_DM_STATUS_RSP 96 +#define TSDB_MSG_TYPE_DM_GRANT 97 +#define TSDB_MSG_TYPE_DM_GRANT_RSP 98 + +#define TSDB_MSG_TYPE_SDB_SYNC 101 +#define TSDB_MSG_TYPE_SDB_SYNC_RSP 102 +#define TSDB_MSG_TYPE_SDB_FORWARD 103 +#define TSDB_MSG_TYPE_SDB_FORWARD_RSP 104 + +#define TSDB_MSG_TYPE_MAX 105 // IE type -#define TSDB_IE_TYPE_SEC 1 -#define TSDB_IE_TYPE_META 2 -#define TSDB_IE_TYPE_MGMT_IP 3 -#define TSDB_IE_TYPE_DNODE_CFG 4 -#define TSDB_IE_TYPE_NEW_VERSION 5 -#define TSDB_IE_TYPE_DNODE_EXT 6 -#define TSDB_IE_TYPE_DNODE_STATE 7 +#define TSDB_IE_TYPE_SEC 1 +#define TSDB_IE_TYPE_META 2 +#define TSDB_IE_TYPE_MGMT_IP 3 +#define TSDB_IE_TYPE_DNODE_CFG 4 +#define TSDB_IE_TYPE_NEW_VERSION 5 +#define TSDB_IE_TYPE_DNODE_EXT 6 +#define TSDB_IE_TYPE_DNODE_STATE 7 enum _mgmt_table { TSDB_MGMT_TABLE_ACCT, @@ -157,46 +157,37 @@ enum _mgmt_table { TSDB_MGMT_TABLE_MAX, }; -#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 -#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 -#define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3 -#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 +#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 +#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 +#define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3 +#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 -#define TSDB_ALTER_TABLE_ADD_COLUMN 5 -#define TSDB_ALTER_TABLE_DROP_COLUMN 6 +#define TSDB_ALTER_TABLE_ADD_COLUMN 5 +#define TSDB_ALTER_TABLE_DROP_COLUMN 6 -#define TSDB_INTERPO_NONE 0 -#define TSDB_INTERPO_NULL 1 -#define TSDB_INTERPO_SET_VALUE 2 -#define TSDB_INTERPO_LINEAR 3 -#define TSDB_INTERPO_PREV 4 +#define TSDB_INTERPO_NONE 0 +#define TSDB_INTERPO_NULL 1 +#define TSDB_INTERPO_SET_VALUE 2 +#define TSDB_INTERPO_LINEAR 3 +#define TSDB_INTERPO_PREV 4 -#define TSDB_ALTER_USER_PASSWD 0x1 -#define TSDB_ALTER_USER_PRIVILEGES 0x2 +#define TSDB_ALTER_USER_PASSWD 0x1 +#define TSDB_ALTER_USER_PRIVILEGES 0x2 -#define TSDB_KILL_MSG_LEN 30 +#define TSDB_KILL_MSG_LEN 30 -#define TSDB_VN_READ_ACCCESS ((char)0x1) +#define TSDB_VN_READ_ACCCESS ((char)0x1) #define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u -#define TSDB_COL_TAG 0x1u -#define TSDB_COL_JOIN 0x2u +#define TSDB_COL_NORMAL 0x0u +#define TSDB_COL_TAG 0x1u +#define TSDB_COL_JOIN 0x2u extern char *taosMsg[]; #pragma pack(push, 1) -typedef struct { - int32_t vnode; - int32_t sid; - int32_t sversion; - uint64_t uid; - int16_t numOfRows; - char payLoad[]; -} SShellSubmitBlock; - typedef struct { int32_t numOfVnodes; } SMsgDesc; @@ -206,27 +197,39 @@ typedef struct SMsgHead { int32_t vgId; } SMsgHead; -typedef struct { - SMsgDesc desc; - SMsgHead header; - int16_t import; - int32_t numOfTables; // total number of sid - char blks[]; // number of data blocks, each table has at least one data block -} SShellSubmitMsg; - -typedef struct { - int32_t index; // index of failed block in submit blocks - int32_t vnode; // vnode index of failed block - int32_t sid; // table index of failed block - int32_t code; // errorcode while write data to vnode, such as not created, dropped, no space, invalid table +// Submit message for one table +typedef struct SSubmitBlk { + int64_t uid; // table unique id + int32_t tid; // table id + int32_t padding; // TODO just for padding here + int32_t sversion; // data schema version + int32_t len; // data part length, not including the SSubmitBlk head + int16_t numOfRows; // total number of rows in current submit block + char data[]; +} SSubmitBlk; + +// Submit message for this TSDB +typedef struct SSubmitMsg { + SMsgHead header; + int32_t length; + int32_t compressed : 2; + int32_t numOfBlocks : 30; + SSubmitBlk blocks[]; +} SSubmitMsg; + +typedef struct { + int32_t index; // index of failed block in submit blocks + int32_t vnode; // vnode index of failed block + int32_t sid; // table index of failed block + int32_t code; // errorcode while write data to vnode, such as not created, dropped, no space, invalid table } SShellSubmitRspBlock; typedef struct { - int32_t code; // 0-success, > 0 error code - int32_t numOfRows; // number of records the client is trying to write - int32_t affectedRows; // number of records actually written - int32_t failedRows; // number of failed records (exclude duplicate records) - int32_t numOfFailedBlocks; + int32_t code; // 0-success, > 0 error code + int32_t numOfRows; // number of records the client is trying to write + int32_t affectedRows; // number of records actually written + int32_t failedRows; // number of failed records (exclude duplicate records) + int32_t numOfFailedBlocks; SShellSubmitRspBlock failedBlocks[]; } SShellSubmitRspMsg; @@ -238,38 +241,38 @@ typedef struct SSchema { } SSchema; typedef struct { - int32_t vnode; //the index of vnode + int32_t vnode; // the index of vnode uint32_t ip; } SVnodeDesc; typedef struct { - int32_t contLen; - int32_t vgId; - int8_t tableType; - int16_t numOfColumns; - int16_t numOfTags; - int32_t sid; - int32_t sversion; - int32_t tagDataLen; - int32_t sqlDataLen; - uint64_t uid; - uint64_t superTableUid; - uint64_t createdTime; - char tableId[TSDB_TABLE_ID_LEN]; - char superTableId[TSDB_TABLE_ID_LEN]; - char data[]; + int32_t contLen; + int32_t vgId; + int8_t tableType; + int16_t numOfColumns; + int16_t numOfTags; + int32_t sid; + int32_t sversion; + int32_t tagDataLen; + int32_t sqlDataLen; + uint64_t uid; + uint64_t superTableUid; + uint64_t createdTime; + char tableId[TSDB_TABLE_ID_LEN]; + char superTableId[TSDB_TABLE_ID_LEN]; + char data[]; } SMDCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - char db[TSDB_DB_NAME_LEN]; - int8_t igExists; - int16_t numOfTags; - int16_t numOfColumns; - int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string - int32_t contLen; - int8_t reserved[16]; - char schema[]; + char tableId[TSDB_TABLE_ID_LEN]; + char db[TSDB_DB_NAME_LEN]; + int8_t igExists; + int16_t numOfTags; + int16_t numOfColumns; + int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string + int32_t contLen; + int8_t reserved[16]; + char schema[]; } SCMCreateTableMsg; typedef struct { @@ -311,7 +314,7 @@ typedef struct { int64_t maxQueryTime; // In unit of hour int64_t maxInbound; int64_t maxOutbound; - int8_t accessState; // Configured only by command + int8_t accessState; // Configured only by command } SAcctCfg; typedef struct { @@ -336,11 +339,11 @@ typedef struct { } SMgmtHead; typedef struct { - int32_t contLen; - int32_t vgId; - int32_t sid; - uint64_t uid; - char tableId[TSDB_TABLE_ID_LEN + 1]; + int32_t contLen; + int32_t vgId; + int32_t sid; + uint64_t uid; + char tableId[TSDB_TABLE_ID_LEN + 1]; } SMDDropTableMsg; typedef struct { @@ -351,7 +354,7 @@ typedef struct { } SMDDropSTableMsg; typedef struct { - int32_t vgId; + int32_t vgId; } SMDDropVnodeMsg; typedef struct SColIndexEx { @@ -366,7 +369,7 @@ typedef struct SColIndexEx { */ int16_t colIdx; int16_t colIdxInBuf; - uint16_t flag; // denote if it is a tag or not + uint16_t flag; // denote if it is a tag or not char name[TSDB_COL_NAME_LEN]; } SColIndexEx; @@ -438,7 +441,7 @@ typedef struct SColumnInfo { typedef struct STableIdInfo { int32_t sid; int64_t uid; - TSKEY key; // last accessed ts, for subscription + TSKEY key; // last accessed ts, for subscription } STableIdInfo; typedef struct STimeWindow { @@ -452,47 +455,31 @@ typedef struct STimeWindow { * the outputCols will be 3 while the numOfCols is 1. */ typedef struct { - int32_t contLen; // msg header - int16_t vgId; - - int32_t numOfTables; - uint64_t uid; + SMsgHead head; STimeWindow window; - - int16_t order; - int16_t orderColId; - - int16_t numOfCols; // the number of columns will be load from vnode - char slidingTimeUnit; // time interval type, for revisement of interval(1d) - - int64_t intervalTime; // time interval for aggregation, in million second - int64_t slidingTime; // value for sliding window - - // tag schema, used to parse tag information in pSidExtInfo - uint64_t pTagSchema; - - int16_t numOfTagsCols; // required number of tags - int16_t tagLength; // tag length in current query - - int16_t numOfGroupCols; // num of group by columns - int16_t orderByIdx; - int16_t orderType; // used in group by xx order by xxx - uint64_t groupbyTagIds; - - int64_t limit; - int64_t offset; - - int16_t queryType; // denote another query process - int16_t numOfOutputCols; // final output columns numbers - - int16_t interpoType; // interpolate type - uint64_t defaultVal; // default value array list - - int32_t colNameLen; - int64_t colNameList; - - int64_t pSqlFuncExprs; - + int32_t numOfTables; + int16_t order; + int16_t orderColId; + int16_t numOfCols; // the number of columns will be load from vnode + int64_t intervalTime; // time interval for aggregation, in million second + int64_t intervalOffset; // start offset for interval query + int64_t slidingTime; // value for sliding window + char slidingTimeUnit; // time interval type, for revisement of interval(1d) + uint16_t tagCondLen; // tag length in current query + uint16_t nameCondLen; // table name in/like query expression string length + int16_t numOfGroupCols; // num of group by columns + int16_t orderByIdx; + int16_t orderType; // used in group by xx order by xxx + uint64_t groupbyTagIds; + int64_t limit; + int64_t offset; + uint16_t queryType; // denote another query process + int16_t numOfOutputCols; // final output columns numbers + int16_t interpoType; // interpolate type + uint64_t defaultVal; // default value array list + + int32_t colNameLen; + int64_t colNameList; int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed int32_t tsLen; // total length of ts comp block int32_t tsNumOfBlocks; // ts comp block numbers @@ -506,12 +493,14 @@ typedef struct { } SQueryTableRsp; typedef struct { + SMsgHead header; uint64_t qhandle; uint16_t free; } SRetrieveTableMsg; -typedef struct { +typedef struct SRetrieveTableRsp { int32_t numOfRows; + int8_t completed; // all results are returned to client int16_t precision; int64_t offset; // updated offset value for multi-vnode projection query int64_t useconds; @@ -593,11 +582,11 @@ typedef struct { char dnodeName[TSDB_DNODE_NAME_LEN]; uint32_t privateIp; uint32_t publicIp; - uint32_t lastReboot; // time stamp for last reboot - uint16_t numOfTotalVnodes; // from config file + uint32_t lastReboot; // time stamp for last reboot + uint16_t numOfTotalVnodes; // from config file uint16_t openVnodes; uint16_t numOfCores; - float diskAvailable; // GB + float diskAvailable; // GB uint8_t alternativeRole; uint8_t reserve[15]; SVnodeLoad load[]; @@ -625,14 +614,14 @@ typedef struct { char tableIds[]; } SCMMultiTableInfoMsg; -typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; -} SCMSuperTableInfoMsg; +typedef struct SCMSTableVgroupMsg { + char tableId[TSDB_TABLE_ID_LEN]; +} SCMSTableVgroupMsg; typedef struct { int32_t numOfDnodes; uint32_t dnodeIps[]; -} SCMSuperTableInfoRsp; +} SCMSTableVgroupRspMsg; typedef struct { int16_t elemLen; @@ -678,25 +667,26 @@ typedef struct { } SSuperTableMeta; typedef struct STableMetaMsg { - char tableId[TSDB_TABLE_ID_LEN]; // note: This field must be at the front - int32_t contLen; - uint8_t numOfTags; - uint8_t precision; - uint8_t tableType; - int16_t numOfColumns; - int16_t sversion; + int32_t contLen; - int8_t numOfVpeers; + char tableId[TSDB_TABLE_ID_LEN]; // table id + char stableId[TSDB_TABLE_ID_LEN]; // stable name if it is created according to super table + uint8_t numOfTags; + uint8_t precision; + uint8_t tableType; + int16_t numOfColumns; + int16_t sversion; + int8_t numOfVpeers; SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT]; - int32_t sid; - int32_t vgId; - uint64_t uid; - SSchema schema[]; + int32_t sid; + int32_t vgId; + uint64_t uid; + SSchema schema[]; } STableMetaMsg; typedef struct SMultiTableMeta { - int32_t numOfTables; - int32_t contLen; + int32_t numOfTables; + int32_t contLen; STableMetaMsg metas[]; } SMultiTableMeta; @@ -733,8 +723,8 @@ typedef struct { } SDMConfigTableMsg; typedef struct { - uint32_t dnode; - int32_t vnode; + uint32_t dnodeId; + int32_t vgId; } SDMConfigVnodeMsg; typedef struct { @@ -761,13 +751,13 @@ typedef struct { } SStreamDesc; typedef struct { - int32_t numOfQueries; - SQueryDesc qdesc[]; + int32_t numOfQueries; + SQueryDesc qdesc[]; } SQqueryList; typedef struct { - int32_t numOfStreams; - SStreamDesc sdesc[]; + int32_t numOfStreams; + SStreamDesc sdesc[]; } SStreamList; typedef struct { diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 66e8cf73988ab25db7544b9a52215d2279630c63..386c8a92f781670f38580cbc794e4e0d3044fce4 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -2,5 +2,5 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(shell) -ADD_SUBDIRECTORY(taosdemo) -ADD_SUBDIRECTORY(taosdump) +#ADD_SUBDIRECTORY(taosdemo) +#ADD_SUBDIRECTORY(taosdump) diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 5bf4cfd604c81f14a95129bcd6887d2ca2904b4f..bacd7497ad5832b963db7d188550e4ce560829c7 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(mnode ${SRC}) - TARGET_LINK_LIBRARIES(mnode trpc tutil sdb pthread) + TARGET_LINK_LIBRARIES(mnode trpc tutil pthread) IF (TD_CLUSTER) TARGET_LINK_LIBRARIES(mnode) diff --git a/src/mnode/inc/mgmtChildTable.h b/src/mnode/inc/mgmtChildTable.h index b16dd58f67402e84c2c611282fded15abea88e97..3f6b1d8b858a8e9bcbee8f59a65ab07b87c99965 100644 --- a/src/mnode/inc/mgmtChildTable.h +++ b/src/mnode/inc/mgmtChildTable.h @@ -23,22 +23,18 @@ extern "C" { #include #include #include "taosdef.h" - #include "mnode.h" int32_t mgmtInitChildTables(); void mgmtCleanUpChildTables(); void * mgmtGetChildTable(char *tableId); -void *mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid); -void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pCreate, SChildTableObj *pTable); - -int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable); -int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName, char *nContent); - -int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); - -void mgmtDropAllChildTables(SDbObj *pDropDb); +void mgmtCreateChildTable(SQueuedMsg *pMsg); +void mgmtDropChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable); +void mgmtGetChildTableMeta(SQueuedMsg *pMsg, SChildTableObj *pTable); +void mgmtAlterChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable); +void mgmtDropAllChildTables(SDbObj *pDropDb); +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtDb.h b/src/mnode/inc/mgmtDb.h index 32bb9d9ec63f386200f85f00d2c5d92c07b04273..27918b9cf94bc2a9591242bc07eaf24fe3d40170 100644 --- a/src/mnode/inc/mgmtDb.h +++ b/src/mnode/inc/mgmtDb.h @@ -22,22 +22,23 @@ extern "C" { #include "mnode.h" -int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup); - +// api int32_t mgmtInitDbs(); void mgmtCleanUpDbs(); SDbObj *mgmtGetDb(char *db); SDbObj *mgmtGetDbByTableId(char *db); bool mgmtCheckIsMonitorDB(char *db, char *monitordb); +// util func void mgmtAddSuperTableIntoDb(SDbObj *pDb); void mgmtRemoveSuperTableFromDb(SDbObj *pDb); void mgmtAddTableIntoDb(SDbObj *pDb); void mgmtRemoveTableFromDb(SDbObj *pDb); +void mgmtAddVgroupIntoDb(SVgObj *pVgroup); +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup); +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup); +void mgmtMoveVgroupToTail(SVgObj *pVgroup); +void mgmtMoveVgroupToHead(SVgObj *pVgroup); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index d768d2dd7cd7ca517a02d5dcce88545b178e2851..8cfe3c7e235f3fabfed0ea403b428d579f2f26da 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -20,9 +20,15 @@ extern "C" { #endif -bool mgmtCheckRedirect(void *handle); +int32_t mgmtInitMnodes(); +void mgmtCleanupMnodes(); -void mgmtGetMnodeIpList(SRpcIpSet *ipSet); +bool mgmtInServerStatus(); +bool mgmtIsMaster(); + +bool mgmtCheckRedirect(void *handle); +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp); int32_t mgmtRemoveMnode(uint32_t privateIp); diff --git a/src/mnode/inc/mgmtNormalTable.h b/src/mnode/inc/mgmtNormalTable.h deleted file mode 100644 index dd09a62bb4bc5f2051d374725e3759ddf5c2dae5..0000000000000000000000000000000000000000 --- a/src/mnode/inc/mgmtNormalTable.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TBASE_MNODE_NORMAL_TABLE_H -#define TBASE_MNODE_NORMAL_TABLE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include "mnode.h" - -int32_t mgmtInitNormalTables(); -void mgmtCleanUpNormalTables(); -void * mgmtGetNormalTable(char *tableId); - -void * mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid); -void * mgmtBuildCreateNormalTableMsg(SNormalTableObj *pTable); - -int32_t mgmtDropNormalTable(SQueuedMsg *newMsg, SNormalTableObj *pTable); -int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int32_t ncols); -int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName); - -int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); - -void mgmtDropAllNormalTables(SDbObj *pDropDb); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/mnode/inc/mgmtProfile.h b/src/mnode/inc/mgmtProfile.h index 7f9fd9622c7b782e1d796fc36f6340198702934b..a40f572f8bc333aafcb8490a36c5e5c290fa5409 100644 --- a/src/mnode/inc/mgmtProfile.h +++ b/src/mnode/inc/mgmtProfile.h @@ -24,9 +24,11 @@ extern "C" { int32_t mgmtInitProfile(); void mgmtCleanUpProfile(); -bool mgmtCheckQhandle(uint64_t qhandle); -void mgmtSaveQhandle(void *qhandle); -void mgmtFreeQhandle(void *qhandle); +bool mgmtCheckQhandle(uint64_t qhandle); +void mgmtSaveQhandle(void *qhandle); +void mgmtFreeQhandle(void *qhandle); + +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h new file mode 100644 index 0000000000000000000000000000000000000000..f291e9341878bedded173840c77d8d61f7da862d --- /dev/null +++ b/src/mnode/inc/mgmtSdb.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_MNODE_SDB_H +#define TDENGINE_MNODE_SDB_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + SDB_KEY_TYPE_STRING, + SDB_KEY_TYPE_AUTO +} ESdbKeyType; + +typedef enum { + SDB_OPER_TYPE_GLOBAL, + SDB_OPER_TYPE_LOCAL +} ESdbOperType; + +typedef struct { + ESdbOperType type; + void * table; + void * pObj; + int64_t version; + int32_t maxRowSize; + int32_t rowSize; + void * rowData; +} SSdbOperDesc; + +typedef struct { + char *tableName; + int32_t hashSessions; + int32_t maxRowSize; + ESdbKeyType keyType; + int32_t (*insertFp)(SSdbOperDesc *pOper); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pDesc); + int32_t (*destroyFp)(SSdbOperDesc *pDesc); +} SSdbTableDesc; + +void * sdbOpenTable(SSdbTableDesc *desc); +void sdbCloseTable(void *handle); + +int32_t sdbInsertRow(SSdbOperDesc *pOper); +int32_t sdbDeleteRow(SSdbOperDesc *pOper); +int32_t sdbUpdateRow(SSdbOperDesc *pOper); + +void *sdbGetRow(void *handle, void *key); +void *sdbFetchRow(void *handle, void *pNode, void **ppRow); +int64_t sdbGetNumOfRows(void *handle); +uint64_t sdbGetVersion(); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/mnode/inc/mgmtShell.h b/src/mnode/inc/mgmtShell.h index 56625b982aed2fa90b39d35be7014ba7ed85e6a4..b92e9de1f463f7e95205ab8c3775d512127e7249 100644 --- a/src/mnode/inc/mgmtShell.h +++ b/src/mnode/inc/mgmtShell.h @@ -22,8 +22,8 @@ extern "C" { #include "mnode.h" int32_t mgmtInitShell(); -void mgmtCleanUpShell(); -void mgmtAddShellMsgHandle(uint8_t msgType, void (*fp)(SQueuedMsg *queuedMsg)); +void mgmtCleanUpShell(); +void mgmtAddShellMsgHandle(uint8_t msgType, void (*fp)(SQueuedMsg *queuedMsg)); typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn); diff --git a/src/mnode/inc/mgmtSuperTable.h b/src/mnode/inc/mgmtSuperTable.h index 922aafed7f9344ad908893cb6c7e6278a541bf35..73809148dec89f0e5c97a59164baa6490d0c4689 100644 --- a/src/mnode/inc/mgmtSuperTable.h +++ b/src/mnode/inc/mgmtSuperTable.h @@ -22,31 +22,20 @@ extern "C" { #include #include - #include "taosdef.h" #include "mnode.h" int32_t mgmtInitSuperTables(); void mgmtCleanUpSuperTables(); - void * mgmtGetSuperTable(char *tableId); -int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate); -int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pTable); -int32_t mgmtAddSuperTableTag(SSuperTableObj *pTable, SSchema schema[], int32_t ntags); -int32_t mgmtDropSuperTableTag(SSuperTableObj *pTable, char *tagName); -int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pTable, char *oldTagName, char *newTagName); -int32_t mgmtAddSuperTableColumn(SSuperTableObj *pTable, SSchema schema[], int32_t ncols); -int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pTable, char *colName); - -int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); -void * mgmtGetSuperTableVgroup(SSuperTableObj *pStable); - -int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pTable, const char *tagName); +void mgmtCreateSuperTable(SQueuedMsg *pMsg); +void mgmtDropSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pTable); +void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable); +void mgmtAlterSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pTable); +void mgmtDropAllSuperTables(SDbObj *pDropDb); int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable); -void mgmtDropAllSuperTables(SDbObj *pDropDb); - #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mgmtTable.h b/src/mnode/inc/mgmtTable.h index b145210e9bbf1800ad6e80b46f635407102fb86c..66557b33621bc1b3d950491009e02cf2799e042a 100644 --- a/src/mnode/inc/mgmtTable.h +++ b/src/mnode/inc/mgmtTable.h @@ -27,13 +27,8 @@ extern "C" { int32_t mgmtInitTables(); void mgmtCleanUpTables(); -STableInfo* mgmtGetTable(char *tableId); - -STableInfo* mgmtGetTableByPos(uint32_t dnodeIp, int32_t vnode, int32_t sid); -int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMetaMsg *pMeta, bool usePublicIp); - -void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable); -void mgmtRemoveTableFromSuperTable(SSuperTableObj *pStable); +STableInfo* mgmtGetTable(char* tableId); +void mgmtExtractTableName(char* tableId, char* tableName); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtUser.h b/src/mnode/inc/mgmtUser.h index d1f927f6de629d9d0fb374deff58ff73cf8baa62..465b8e4f96e42cc8148c478c9ce87b0750f6f606 100644 --- a/src/mnode/inc/mgmtUser.h +++ b/src/mnode/inc/mgmtUser.h @@ -24,7 +24,7 @@ extern "C" { int32_t mgmtInitUsers(); void mgmtCleanUpUsers(); SUserObj *mgmtGetUser(char *name); -SUserObj *mgmtGetUserFromConn(void *pConn); +SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index b7c68b5f8062240fd1f7c71f1e789d3a31f08630..de0648da44bfec3d007fdcdf565c34dd46022650 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -27,18 +27,15 @@ extern "C" { int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); -SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode); +void mgmtDropAllVgroups(SDbObj *pDropDb); void mgmtCreateVgroup(SQueuedMsg *pMsg); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); -void mgmtUpdateVgroup(SVgObj *pVgroup); -void mgmtUpdateVgroupIp(SDnodeObj *pDnode); - -void mgmtSetVgroupIdPool(); +void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle); SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb); -void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable); -void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable); +void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable); +void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable); void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle); void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle); diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c index b1ee72386ef9bc36635c982685e7885bd1ab5f93..0075f53319cf63f92457c02204d4d1ab098706c5 100644 --- a/src/mnode/src/mgmtAcct.c +++ b/src/mnode/src/mgmtAcct.c @@ -85,8 +85,10 @@ int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) { pUser->prev->next = pUser->next; } - if (pUser->next) pUser->next->prev = pUser->prev; - + if (pUser->next) { + pUser->next->prev = pUser->prev; + } + if (pUser->prev == NULL) { pAcct->pUser = pUser->next; } diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index d5f0e7c85378d2b775a323a3d1a1bad783088a3f..4936c941ca61d987972f83c330128217e7dbe4a7 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -21,169 +21,176 @@ #include "ttime.h" #include "tstatus.h" #include "tutil.h" +#include "qast.h" +#include "qextbuffer.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "tscompression.h" +#include "tskiplist.h" +#include "tsqlfunction.h" +#include "tstatus.h" +#include "ttime.h" +#include "name.h" #include "mnode.h" #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDb.h" +#include "mgmtDClient.h" +#include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtGrant.h" +#include "mgmtMnode.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" -#include "mgmtDClient.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtVgroup.h" - -void *tsChildTableSdb; -int32_t tsChildTableUpdateSize; -void *(*mgmtChildTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); - -void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); +#include "mgmtUser.h" + +static void *tsChildTableSdb; +static int32_t tsChildTableUpdateSize; +static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *queueMsg); +static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg); +static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg); +static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg); +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg); +static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtDestroyChildTable(SChildTableObj *pTable) { - free(pTable); + tfree(pTable->schema); + tfree(pTable->sql); + tfree(pTable); } -static void mgmtChildTableActionInit() { - mgmtChildTableActionFp[SDB_TYPE_INSERT] = mgmtChildTableActionInsert; - mgmtChildTableActionFp[SDB_TYPE_DELETE] = mgmtChildTableActionDelete; - mgmtChildTableActionFp[SDB_TYPE_UPDATE] = mgmtChildTableActionUpdate; - mgmtChildTableActionFp[SDB_TYPE_ENCODE] = mgmtChildTableActionEncode; - mgmtChildTableActionFp[SDB_TYPE_DECODE] = mgmtChildTableActionDecode; - mgmtChildTableActionFp[SDB_TYPE_RESET] = mgmtChildTableActionReset; - mgmtChildTableActionFp[SDB_TYPE_DESTROY] = mgmtChildTableActionDestroy; -} - -void *mgmtChildTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; - memcpy(pTable, str, tsChildTableUpdateSize); - return NULL; -} - -void *mgmtChildTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *)row; - mgmtDestroyChildTable(pTable); - return NULL; +static int32_t mgmtChildTableActionDestroy(SSdbOperDesc *pOper) { + mgmtDestroyChildTable(pOper->pObj); + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; +static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("ctable:%s, not in vgroup:%d", pTable->tableId, pTable->vgId); - return NULL; + mError("ctable:%s, not in vgroup:%d", pTable->info.tableId, pTable->vgId); + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("ctable:%s, vgroup:%d not in db:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return NULL; + mError("ctable:%s, vgroup:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { - mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return NULL; + mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct); + return TSDB_CODE_INVALID_ACCT; } - if (!sdbMaster) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid != pTable->sid) { - mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); - return NULL; - } + if (pTable->info.type == TSDB_CHILD_TABLE) { + pTable->superTable = mgmtGetSuperTable(pTable->superTableId); + pTable->superTable->numOfTables++; + mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); + } else { + mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); } - - pTable->superTable = mgmtGetSuperTable(pTable->superTableId); - mgmtAddTableIntoSuperTable(pTable->superTable); - - mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); mgmtAddTableIntoDb(pDb); - mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); - } + mgmtAddTableIntoVgroup(pVgroup, pTable); - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; +static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("ctable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return NULL; + mError("ctable:%s, vgroup:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { - mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return NULL; + mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct); + return TSDB_CODE_INVALID_ACCT; } - mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); - mgmtRemoveTableFromDb(pDb); - mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - - mgmtRemoveTableFromSuperTable(pTable->superTable); - - if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pDb, pVgroup); + if (pTable->info.type == TSDB_CHILD_TABLE) { + mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); + pTable->superTable->numOfTables--; + } else { + mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1); } - - return NULL; + mgmtRemoveTableFromDb(pDb); + mgmtRemoveTableFromVgroup(pVgroup, pTable); + + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtChildTableActionReset(row, str, size, NULL); +static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; - assert(row != NULL && str != NULL); - - memcpy(str, pTable, tsChildTableUpdateSize); - *ssize = tsChildTableUpdateSize; +static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; + assert(pTable != NULL && pOper->rowData != NULL); + + if (pTable->info.type == TSDB_CHILD_TABLE) { + memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); + pOper->rowSize = tsChildTableUpdateSize; + } else { + int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); + if (pOper->maxRowSize < tsChildTableUpdateSize + schemaSize) { + return TSDB_CODE_INVALID_MSG_LEN; + } + memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); + memcpy(pOper->rowData + tsChildTableUpdateSize, pTable->schema, schemaSize); + memcpy(pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); + pOper->rowSize = tsChildTableUpdateSize + schemaSize + pTable->sqlLen; + } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); - - SChildTableObj *pTable = (SChildTableObj *)calloc(sizeof(SChildTableObj), 1); - if (pTable == NULL) return NULL; - - if (size < tsChildTableUpdateSize) { - mgmtDestroyChildTable(pTable); - return NULL; +static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); + SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); + if (pTable == NULL) { + return TSDB_CODE_SERV_OUT_OF_MEMORY; } - memcpy(pTable, str, tsChildTableUpdateSize); - return (void *)pTable; -} + memcpy(pTable, pOper->rowData, tsChildTableUpdateSize); -void *mgmtChildTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtChildTableActionFp[(uint8_t)action] != NULL) { - return (*(mgmtChildTableActionFp[(uint8_t)action]))(row, str, size, ssize); + if (pTable->info.type != TSDB_CHILD_TABLE) { + int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); + pTable->schema = (SSchema *)malloc(schemaSize); + if (pTable->schema == NULL) { + mgmtDestroyChildTable(pTable); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + memcpy(pTable->schema, pOper->rowData + tsChildTableUpdateSize, schemaSize); + + pTable->sql = (char *)malloc(pTable->sqlLen); + if (pTable->sql == NULL) { + mgmtDestroyChildTable(pTable); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + memcpy(pTable->sql, pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sqlLen); } - return NULL; + + pOper->pObj = pTable; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitChildTables() { @@ -191,12 +198,23 @@ int32_t mgmtInitChildTables() { void *pLastNode = NULL; SChildTableObj *pTable = NULL; - mgmtChildTableActionInit(); SChildTableObj tObj; - tsChildTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "ctables", + .hashSessions = tsMaxTables, + .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtChildTableActionInsert, + .deleteFp = mgmtChildTableActionDelete, + .updateFp = mgmtChildTableActionUpdate, + .encodeFp = mgmtChildTableActionEncode, + .decodeFp = mgmtChildTableActionDecode, + .destroyFp = mgmtChildTableActionDestroy, + }; - tsChildTableSdb = sdbOpenTable(tsMaxTables, tsChildTableUpdateSize, - "ctables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtChildTableAction); + tsChildTableSdb = sdbOpenTable(&tableDesc); if (tsChildTableSdb == NULL) { mError("failed to init child table data"); return -1; @@ -204,64 +222,86 @@ int32_t mgmtInitChildTables() { pNode = NULL; while (1) { + pLastNode = pNode; pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); if (pTable == NULL) { break; } - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); + SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId); if (pDb == NULL) { - mError("ctable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsChildTableSdb, pTable); + mError("ctable:%s, failed to get db, discard it", pTable->info.tableId); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); + mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } if (strcmp(pVgroup->dbName, pDb->name) != 0) { mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", - pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); + pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } if (pVgroup->tableList == NULL) { - mError("ctable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); + mError("ctable:%s, vgroup:%d tableList is null", pTable->info.tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } - pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1); - - SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTable->superTableId); - if (pSuperTable == NULL) { - mError("ctable:%s, stable:%s not exist", pTable->tableId, pTable->superTableId); - pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); - pNode = pLastNode; - continue; + if (pTable->info.type == TSDB_CHILD_TABLE) { + SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTable->superTableId); + if (pSuperTable == NULL) { + mError("ctable:%s, stable:%s not exist", pTable->info.tableId, pTable->superTableId); + pTable->vgId = 0; + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); + pNode = pLastNode; + continue; + } } - - pTable->superTable = pSuperTable; - mgmtAddTableIntoSuperTable(pSuperTable); - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); } + mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_TABLES_META, mgmtProcessMultiTableMetaMsg); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mgmtGetShowTableMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mgmtRetrieveShowTables); + mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP, mgmtProcessCreateTableRsp); + mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropTableRsp); + mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_TABLE, mgmtProcessTableCfgMsg); + mTrace("child table is initialized"); return 0; } @@ -270,11 +310,20 @@ void mgmtCleanUpChildTables() { sdbCloseTable(tsChildTableSdb); } -void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { - char *pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; - int32_t tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; - int32_t totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags; - int32_t contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen; +static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { + char * pTagData = NULL; + int32_t tagDataLen = 0; + int32_t totalCols = 0; + int32_t contLen = 0; + if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) { + pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; + tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; + totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags; + contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen + pTable->sqlLen; + } else { + totalCols = pTable->numOfColumns; + contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + pTable->sqlLen; + } SMDCreateTableMsg *pCreate = rpcMallocCont(contLen); if (pCreate == NULL) { @@ -282,91 +331,207 @@ void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTab return NULL; } - memcpy(pCreate->tableId, pTable->tableId, TSDB_TABLE_ID_LEN + 1); - memcpy(pCreate->superTableId, pTable->superTable->tableId, TSDB_TABLE_ID_LEN + 1); + memcpy(pCreate->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN + 1); pCreate->contLen = htonl(contLen); pCreate->vgId = htonl(pTable->vgId); - pCreate->tableType = pTable->type; - pCreate->numOfColumns = htons(pTable->superTable->numOfColumns); - pCreate->numOfTags = htons(pTable->superTable->numOfTags); + pCreate->tableType = pTable->info.type; + pCreate->createdTime = htobe64(pTable->createdTime); pCreate->sid = htonl(pTable->sid); - pCreate->sversion = htonl(pTable->superTable->sversion); - pCreate->tagDataLen = htonl(tagDataLen); - pCreate->sqlDataLen = 0; + pCreate->sqlDataLen = htonl(pTable->sqlLen); pCreate->uid = htobe64(pTable->uid); - pCreate->superTableUid = htobe64(pTable->superTable->uid); - pCreate->createdTime = htobe64(pTable->createdTime); - + + if (pTable->info.type == TSDB_CHILD_TABLE) { + memcpy(pCreate->superTableId, pTable->superTable->info.tableId, TSDB_TABLE_ID_LEN + 1); + pCreate->numOfColumns = htons(pTable->superTable->numOfColumns); + pCreate->numOfTags = htons(pTable->superTable->numOfTags); + pCreate->sversion = htonl(pTable->superTable->sversion); + pCreate->tagDataLen = htonl(tagDataLen); + pCreate->superTableUid = htobe64(pTable->superTable->uid); + } else { + pCreate->numOfColumns = htons(pTable->numOfColumns); + pCreate->numOfTags = 0; + pCreate->sversion = htonl(pTable->sversion); + pCreate->tagDataLen = 0; + pCreate->superTableUid = 0; + } + SSchema *pSchema = (SSchema *) pCreate->data; - memcpy(pSchema, pTable->superTable->schema, totalCols * sizeof(SSchema)); + if (pTable->info.type == TSDB_CHILD_TABLE) { + memcpy(pSchema, pTable->superTable->schema, totalCols * sizeof(SSchema)); + } else { + memcpy(pSchema, pTable->schema, totalCols * sizeof(SSchema)); + } for (int32_t col = 0; col < totalCols; ++col) { pSchema->bytes = htons(pSchema->bytes); pSchema->colId = htons(pSchema->colId); pSchema++; } - memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); - return pCreate; -} - -void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { - int32_t numOfTables = sdbGetNumOfRows(tsChildTableSdb); - if (numOfTables >= tsMaxTables) { - mError("ctable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, tsMaxTables); - terrno = TSDB_CODE_TOO_MANY_TABLES; - return NULL; + if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) { + memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); + memcpy(pCreate->data + totalCols * sizeof(SSchema) + tagDataLen, pTable->sql, pTable->sqlLen); } - char *pTagData = (char *) pCreate->schema; // it is a tag key - SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData); - if (pSuperTable == NULL) { - mError("ctable:%s, corresponding super table does not exist", pCreate->tableId); - terrno = TSDB_CODE_INVALID_TABLE; - return NULL; - } + return pCreate; +} - SChildTableObj *pTable = (SChildTableObj *) calloc(sizeof(SChildTableObj), 1); +static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { + SChildTableObj *pTable = (SChildTableObj *) calloc(1, sizeof(SChildTableObj)); if (pTable == NULL) { mError("ctable:%s, failed to alloc memory", pCreate->tableId); terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; return NULL; } - strcpy(pTable->tableId, pCreate->tableId); - strcpy(pTable->superTableId, pSuperTable->tableId); - pTable->type = TSDB_CHILD_TABLE; + if (pCreate->numOfColumns == 0) { + pTable->info.type = TSDB_CHILD_TABLE; + } else { + pTable->info.type = TSDB_NORMAL_TABLE; + } + + strcpy(pTable->info.tableId, pCreate->tableId); pTable->createdTime = taosGetTimestampMs(); - pTable->uid = (((uint64_t) pTable->vgId) << 40) + ((((uint64_t) pTable->sid) & ((1ul << 24) - 1ul)) << 16) + - ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sid = tid; pTable->vgId = pVgroup->vgId; - pTable->superTable = pSuperTable; + + if (pTable->info.type == TSDB_CHILD_TABLE) { + char *pTagData = (char *) pCreate->schema; // it is a tag key + SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData); + if (pSuperTable == NULL) { + mError("ctable:%s, corresponding super table does not exist", pCreate->tableId); + free(pTable); + terrno = TSDB_CODE_INVALID_TABLE; + return NULL; + } - if (sdbInsertRow(tsChildTableSdb, pTable, 0) < 0) { + strcpy(pTable->superTableId, pSuperTable->info.tableId); + pTable->uid = (((uint64_t) pTable->vgId) << 40) + ((((uint64_t) pTable->sid) & ((1ul << 24) - 1ul)) << 16) + + (sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->superTable = pSuperTable; + } else { + pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->sversion = 0; + pTable->numOfColumns = htons(pCreate->numOfColumns); + pTable->sqlLen = htons(pCreate->sqlLen); + + int32_t numOfCols = pTable->numOfColumns; + int32_t schemaSize = numOfCols * sizeof(SSchema); + pTable->schema = (SSchema *) calloc(1, schemaSize); + if (pTable->schema == NULL) { + free(pTable); + terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; + return NULL; + } + memcpy(pTable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); + + pTable->nextColId = 0; + for (int32_t col = 0; col < numOfCols; col++) { + SSchema *tschema = pTable->schema; + tschema[col].colId = pTable->nextColId++; + tschema[col].bytes = htons(tschema[col].bytes); + } + + if (pTable->sqlLen != 0) { + pTable->info.type = TSDB_STREAM_TABLE; + pTable->sql = calloc(1, pTable->sqlLen); + if (pTable->sql == NULL) { + free(pTable); + terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; + return NULL; + } + memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); + pTable->sql[pTable->sqlLen - 1] = 0; + mTrace("table:%s, stream sql len:%d sql:%s", pTable->info.tableId, pTable->sqlLen, pTable->sql); + } + } + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + + if (sdbInsertRow(&desc) != TSDB_CODE_SUCCESS) { free(pTable); mError("ctable:%s, update sdb error", pCreate->tableId); terrno = TSDB_CODE_SDB_ERROR; return NULL; } - mTrace("ctable:%s, create ctable in vgroup, uid:%" PRIu64 , pTable->tableId, pTable->uid); + mTrace("ctable:%s, create ctable in vgroup, uid:%" PRIu64 , pTable->info.tableId, pTable->uid); return pTable; } -int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable) { +void mgmtCreateChildTable(SQueuedMsg *pMsg) { + SCMCreateTableMsg *pCreate = pMsg->pCont; + + int32_t code = mgmtCheckTimeSeries(htons(pCreate->numOfColumns)); + if (code != TSDB_CODE_SUCCESS) { + mError("table:%s, failed to create, timeseries exceed the limit", pCreate->tableId); + mgmtSendSimpleResp(pMsg->thandle, code); + return; + } + + SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); + memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); + pMsg->pCont = NULL; + + SVgObj *pVgroup = mgmtGetAvailableVgroup(pMsg->pDb); + if (pVgroup == NULL) { + mTrace("table:%s, start to create a new vgroup", pCreate->tableId); + mgmtCreateVgroup(newMsg); + return; + } + + int32_t sid = taosAllocateId(pVgroup->idPool); + if (sid < 0) { + mTrace("tables:%s, no enough sid in vgroup:%d", pVgroup->vgId); + mgmtCreateVgroup(newMsg); + return; + } + + SChildTableObj *pTable = mgmtDoCreateChildTable(pCreate, pVgroup, sid); + if (pTable == NULL) { + mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); + return; + } + + SMDCreateTableMsg *pMDCreate = mgmtBuildCreateChildTableMsg(pCreate, (SChildTableObj *) pTable); + if (pMDCreate == NULL) { + mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); + return; + } + + SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); + SRpcMsg rpcMsg = { + .handle = newMsg, + .pCont = pMDCreate, + .contLen = htonl(pMDCreate->contLen), + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE + }; + + newMsg->ahandle = pTable; + mgmtSendMsgToDnode(&ipSet, &rpcMsg); +} + +void mgmtDropChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable) { SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("ctable:%s, failed to drop child table, vgroup not exist", pTable->tableId); - return TSDB_CODE_OTHERS; + mError("ctable:%s, failed to drop child table, vgroup not exist", pTable->info.tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); + return; } SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg)); if (pDrop == NULL) { - mError("ctable:%s, failed to drop child table, no enough memory", pTable->tableId); - return TSDB_CODE_SERV_OUT_OF_MEMORY; + mError("ctable:%s, failed to drop child table, no enough memory", pTable->info.tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; } - strcpy(pDrop->tableId, pTable->tableId); + strcpy(pDrop->tableId, pTable->info.tableId); pDrop->vgId = htonl(pTable->vgId); pDrop->contLen = htonl(sizeof(SMDDropTableMsg)); pDrop->sid = htonl(pTable->sid); @@ -376,17 +541,15 @@ int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable) { mTrace("ctable:%s, send drop table msg", pDrop->tableId); SRpcMsg rpcMsg = { - .handle = newMsg, + .handle = pMsg, .pCont = pDrop, .contLen = sizeof(SMDDropTableMsg), .code = 0, .msgType = TSDB_MSG_TYPE_MD_DROP_TABLE }; - newMsg->ahandle = pTable; + pMsg->ahandle = pTable; mgmtSendMsgToDnode(&ipSet, &rpcMsg); - - return TSDB_CODE_SUCCESS; } void* mgmtGetChildTable(char *tableId) { @@ -401,7 +564,7 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName // } // // //TODO send msg to dnode -// mTrace("Succeed to modify tag column %d of table %s", col, pTable->tableId); +// mTrace("Succeed to modify tag column %d of table %s", col, pTable->info.tableId); // return TSDB_CODE_SUCCESS; // int32_t rowSize = 0; @@ -433,44 +596,223 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName // if (pTable->isDirty) pTable->isDirty = 0; // // if (ret < 0) { -// mError("Failed to modify tag column %d of table %s", col, pTable->tableId); +// mError("Failed to modify tag column %d of table %s", col, pTable->info.tableId); // return TSDB_CODE_APP_ERROR; // } // -// mTrace("Succeed to modify tag column %d of table %s", col, pTable->tableId); +// mTrace("Succeed to modify tag column %d of table %s", col, pTable->info.tableId); // return TSDB_CODE_SUCCESS; return 0; } -int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { - pMeta->uid = htobe64(pTable->uid); - pMeta->sid = htonl(pTable->sid); - pMeta->vgId = htonl(pTable->vgId); - pMeta->sversion = htons(pTable->superTable->sversion); - pMeta->precision = pDb->cfg.precision; - pMeta->numOfTags = pTable->superTable->numOfTags; - pMeta->numOfColumns = htons(pTable->superTable->numOfColumns); - pMeta->tableType = pTable->type; - pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); - strncpy(pMeta->tableId, pTable->tableId, tListLen(pTable->tableId)); +static int32_t mgmtFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) { + SSchema *schema = (SSchema *) pTable->schema; + for (int32_t i = 0; i < pTable->numOfColumns; i++) { + if (strcasecmp(schema[i].name, colName) == 0) { + return i; + } + } + + return -1; +} + +static int32_t mgmtAddNormalTableColumn(SChildTableObj *pTable, SSchema schema[], int32_t ncols) { + if (ncols <= 0) { + return TSDB_CODE_APP_ERROR; + } + + for (int32_t i = 0; i < ncols; i++) { + if (mgmtFindNormalTableColumnIndex(pTable, schema[i].name) > 0) { + return TSDB_CODE_APP_ERROR; + } + } + + SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId); + if (pDb == NULL) { + mError("table: %s not belongs to any database", pTable->info.tableId); + return TSDB_CODE_APP_ERROR; + } + + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + if (pAcct == NULL) { + mError("DB: %s not belongs to andy account", pDb->name); + return TSDB_CODE_APP_ERROR; + } + + int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); + pTable->schema = realloc(pTable->schema, schemaSize + sizeof(SSchema) * ncols); + + memcpy(pTable->schema + schemaSize, schema, sizeof(SSchema) * ncols); + + SSchema *tschema = (SSchema *) (pTable->schema + sizeof(SSchema) * pTable->numOfColumns); + for (int32_t i = 0; i < ncols; i++) { + tschema[i].colId = pTable->nextColId++; + } + + pTable->numOfColumns += ncols; + pTable->sversion++; + pAcct->acctInfo.numOfTimeSeries += ncols; + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + desc.rowData = pTable; + desc.rowSize = tsChildTableUpdateSize; + sdbUpdateRow(&desc); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtDropNormalTableColumnByName(SChildTableObj *pTable, char *colName) { + int32_t col = mgmtFindNormalTableColumnIndex(pTable, colName); + if (col < 0) { + return TSDB_CODE_APP_ERROR; + } + + SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId); + if (pDb == NULL) { + mError("table: %s not belongs to any database", pTable->info.tableId); + return TSDB_CODE_APP_ERROR; + } + + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + if (pAcct == NULL) { + mError("DB: %s not belongs to any account", pDb->name); + return TSDB_CODE_APP_ERROR; + } + + memmove(pTable->schema + sizeof(SSchema) * col, pTable->schema + sizeof(SSchema) * (col + 1), + sizeof(SSchema) * (pTable->numOfColumns - col - 1)); + + pTable->numOfColumns--; + pTable->sversion++; + + pAcct->acctInfo.numOfTimeSeries--; + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + desc.rowData = pTable; + desc.rowSize = tsChildTableUpdateSize; + sdbUpdateRow(&desc); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pTable) { + int32_t numOfCols = pTable->numOfColumns; + for (int32_t i = 0; i < numOfCols; ++i) { + strcpy(pSchema->name, pTable->schema[i].name); + pSchema->type = pTable->schema[i].type; + pSchema->bytes = htons(pTable->schema[i].bytes); + pSchema->colId = htons(pTable->schema[i].colId); + pSchema++; + } + + return numOfCols * sizeof(SSchema); +} + +static int32_t mgmtDoGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { + pMeta->uid = htobe64(pTable->uid); + pMeta->sid = htonl(pTable->sid); + pMeta->vgId = htonl(pTable->vgId); + pMeta->precision = pDb->cfg.precision; + pMeta->tableType = pTable->info.type; + strncpy(pMeta->tableId, pTable->info.tableId, tListLen(pTable->info.tableId)); + + if (pTable->info.type == TSDB_CHILD_TABLE) { + pMeta->sversion = htons(pTable->superTable->sversion); + pMeta->numOfTags = 0; + pMeta->numOfColumns = htons((int16_t)pTable->superTable->numOfColumns); + pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); + strncpy(pMeta->stableId, pTable->superTable->info.tableId, tListLen(pMeta->stableId)); + } else { + pMeta->sversion = htons(pTable->sversion); + pMeta->numOfTags = 0; + pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); + pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable); + } + SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return TSDB_CODE_INVALID_TABLE; + mError("table:%s, failed to get table meta, db not selected", pTable->info.tableId); + return TSDB_CODE_INVALID_VGROUP_ID; } + for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) { if (usePublicIp) { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; + pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; } else { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp; + pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp; } pMeta->vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); } pMeta->numOfVpeers = pVgroup->numOfVnodes; + mTrace("table:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid); + return TSDB_CODE_SUCCESS; } +void mgmtGetChildTableMeta(SQueuedMsg *pMsg, SChildTableObj *pTable) { + SCMTableInfoMsg *pInfo = pMsg->pCont; + SDbObj *pDb = mgmtGetDbByTableId(pInfo->tableId); + if (pDb == NULL || pDb->dirty) { + mError("table:%s, failed to get table meta, db not selected", pInfo->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); + return; + } + + if (pTable == NULL) { + if (htons(pInfo->createFlag) != 1) { + mError("table:%s, failed to get table meta, table not exist", pInfo->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + return; + } else { + //TODO: on demand create table from super table if table does not exists + int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData); + SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen); + if (pCreateMsg == NULL) { + mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; + } + memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData)); + strcpy(pCreateMsg->tableId, pInfo->tableId); + + SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); + memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); + pMsg->pCont = NULL; + + newMsg->ahandle = newMsg->pCont; + newMsg->pCont = pCreateMsg; + mTrace("table:%s, start to create in demand", pInfo->tableId); + mgmtAddToShellQueue(newMsg); + return; + } + } + + STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); + if (pMeta == NULL) { + mError("table:%s, failed to get table meta, no enough memory", pTable->info.tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; + } + + mgmtDoGetChildTableMeta(pDb, pTable, pMeta, pMsg->usePublicIp); + + SRpcMsg rpcRsp = { + .handle = pMsg->thandle, + .pCont = pMeta, + .contLen = pMeta->contLen, + }; + pMeta->contLen = htons(pMeta->contLen); + rpcSendResponse(&rpcRsp); +} + void mgmtDropAllChildTables(SDbObj *pDropDb) { void *pNode = NULL; void *pLastNode = NULL; @@ -484,13 +826,397 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { break; } - if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsChildTableSdb, pTable); + if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsChildTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); + pNode = pLastNode; + numOfTables++; + continue; + } + } + + mTrace("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); +} + +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { + void *pNode = NULL; + void *pLastNode = NULL; + int32_t numOfTables = 0; + SChildTableObj *pTable = NULL; + + while (1) { + pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); + if (pTable == NULL) { + break; + } + + if (pTable->superTable == pStable) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsChildTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables ++; + numOfTables++; + continue; + } + } + + mTrace("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); +} + +static STableInfo* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) { + SDnodeObj *pObj = mgmtGetDnode(dnodeId); + SVgObj *pVgroup = mgmtGetVgroup(vnode); + + if (pObj == NULL || pVgroup == NULL) { + return NULL; + } + + return (STableInfo *)pVgroup->tableList[sid]; +} + +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont; + pCfg->dnode = htonl(pCfg->dnode); + pCfg->vnode = htonl(pCfg->vnode); + pCfg->sid = htonl(pCfg->sid); + mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + + STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); + if (pTable == NULL) { + mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_TABLE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SMDCreateTableMsg *pMDCreate = NULL; + pMDCreate = mgmtBuildCreateChildTableMsg(NULL, (SChildTableObj *) pTable); + if (pMDCreate == NULL) { + return; + } + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + SRpcMsg rpcRsp = { + .handle = NULL, + .pCont = pMDCreate, + .contLen = htonl(pMDCreate->contLen), + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE + }; + mgmtSendMsgToDnode(&ipSet, &rpcRsp); +} + +static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { + if (rpcMsg->handle == NULL) return; + + SQueuedMsg *queueMsg = rpcMsg->handle; + queueMsg->received++; + + SChildTableObj *pTable = queueMsg->ahandle; + mTrace("table:%s, drop table rsp received, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, tstrerror(rpcMsg->code)); + + if (rpcMsg->code != TSDB_CODE_SUCCESS) { + mError("table:%s, failed to drop in dnode, reason:%s", pTable->info.tableId, tstrerror(rpcMsg->code)); + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + free(queueMsg); + return; + } + + SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); + if (pVgroup == NULL) { + mError("table:%s, failed to get vgroup", pTable->info.tableId); + mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_INVALID_VGROUP_ID); + free(queueMsg); + return; + } + + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mError("table:%s, update ctables sdb error", pTable->info.tableId); + mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); + free(queueMsg); + return; + } + + if (pVgroup->numOfTables <= 0) { + mPrint("vgroup:%d, all tables is dropped, drop vgroup", pVgroup->vgId); + mgmtDropVgroup(pVgroup, NULL); + } + + mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SUCCESS); + free(queueMsg); +} + +static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { + if (rpcMsg->handle == NULL) return; + + SQueuedMsg *queueMsg = rpcMsg->handle; + queueMsg->received++; + + SChildTableObj *pTable = queueMsg->ahandle; + mTrace("table:%s, create table rsp received, thandle:%p ahandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, + rpcMsg->handle, tstrerror(rpcMsg->code)); + + if (rpcMsg->code != TSDB_CODE_SUCCESS) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); + + mError("table:%s, failed to create in dnode, reason:%s", pTable->info.tableId, tstrerror(rpcMsg->code)); + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + } else { + mTrace("table:%s, created in dnode", pTable->info.tableId); + if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) { + SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); + newMsg->msgType = queueMsg->msgType; + newMsg->thandle = queueMsg->thandle; + newMsg->pDb = queueMsg->pDb; + newMsg->pUser = queueMsg->pUser; + newMsg->contLen = queueMsg->contLen; + newMsg->pCont = rpcMallocCont(newMsg->contLen); + memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); + mTrace("table:%s, start to get meta", pTable->info.tableId); + mgmtAddToShellQueue(newMsg); + } else { + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + } + } + + free(queueMsg); +} + +static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg) { + mTrace("alter table rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); +} + +static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { + SRpcConnInfo connInfo; + if (rpcGetConnInfo(pMsg->thandle, &connInfo) != 0) { + mError("conn:%p is already released while get mulit table meta", pMsg->thandle); + return; + } + + bool usePublicIp = (connInfo.serverIp == tsPublicIpInt); + SUserObj *pUser = mgmtGetUser(connInfo.user); + if (pUser == NULL) { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER); + return; + } + + SCMMultiTableInfoMsg *pInfo = pMsg->pCont; + pInfo->numOfTables = htonl(pInfo->numOfTables); + + int32_t totalMallocLen = 4*1024*1024; // first malloc 4 MB, subsequent reallocation as twice + SMultiTableMeta *pMultiMeta = rpcMallocCont(totalMallocLen); + if (pMultiMeta == NULL) { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; + } + + pMultiMeta->contLen = sizeof(SMultiTableMeta); + pMultiMeta->numOfTables = 0; + + for (int t = 0; t < pInfo->numOfTables; ++t) { + char *tableId = (char*)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN); + SChildTableObj *pTable = mgmtGetChildTable(tableId); + if (pTable == NULL) continue; + + SDbObj *pDb = mgmtGetDbByTableId(tableId); + if (pDb == NULL) continue; + + int availLen = totalMallocLen - pMultiMeta->contLen; + if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS) { + //TODO realloc + //totalMallocLen *= 2; + //pMultiMeta = rpcReMalloc(pMultiMeta, totalMallocLen); + //if (pMultiMeta == NULL) { + /// rpcSendResponse(ahandle, TSDB_CODE_SERV_OUT_OF_MEMORY, NULL, 0); + // return TSDB_CODE_SERV_OUT_OF_MEMORY; + //} else { + // t--; + // continue; + //} + } + + STableMetaMsg *pMeta = (STableMetaMsg *)(pMultiMeta->metas + pMultiMeta->contLen); + int32_t code = mgmtDoGetChildTableMeta(pDb, pTable, pMeta, usePublicIp); + if (code == TSDB_CODE_SUCCESS) { + pMultiMeta->numOfTables ++; + pMultiMeta->contLen += pMeta->contLen; + } + } + + SRpcMsg rpcRsp = {0}; + rpcRsp.handle = pMsg->thandle; + rpcRsp.pCont = pMultiMeta; + rpcRsp.contLen = pMultiMeta->contLen; + rpcSendResponse(&rpcRsp); +} + +static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { + SDbObj *pDb = mgmtGetDb(pShow->db); + if (pDb == NULL) { + return TSDB_CODE_DB_NOT_SELECTED; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "table name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "columns"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "stable name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = pDb->numOfTables; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + + return 0; +} + +static void mgmtVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { + if (rows < capacity) { + for (int32_t i = 0; i < numOfCols; ++i) { + memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows); + } + } +} + +static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) { + SDbObj *pDb = mgmtGetDb(pShow->db); + if (pDb == NULL) return 0; + + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); + if (pUser == NULL) return 0; + + if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { + if (strcmp(pUser->user, "root") != 0 && strcmp(pUser->user, "_root") != 0 && + strcmp(pUser->user, "monitor") != 0) { + return 0; + } + } + + int32_t numOfRows = 0; + SChildTableObj *pTable = NULL; + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + char prefix[64] = {0}; + strcpy(prefix, pDb->name); + strcat(prefix, TS_PATH_DELIMITER); + int32_t prefixLen = strlen(prefix); + + while (numOfRows < rows) { + pShow->pNode = sdbFetchRow(tsChildTableSdb, pShow->pNode, (void **) &pTable); + if (pTable == NULL) break; + + // not belong to current db + if (strncmp(pTable->info.tableId, prefix, prefixLen)) { continue; } + + char tableName[TSDB_TABLE_NAME_LEN] = {0}; + memset(tableName, 0, tListLen(tableName)); + + // pattern compare for meter name + mgmtExtractTableName(pTable->info.tableId, tableName); + + if (pShow->payloadLen > 0 && + patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) { + continue; + } + + int32_t cols = 0; + + char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strncpy(pWrite, tableName, TSDB_TABLE_NAME_LEN); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *) pWrite = pTable->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + if (pTable->info.type == TSDB_CHILD_TABLE) { + *(int16_t *)pWrite = pTable->superTable->numOfColumns; + } else { + *(int16_t *)pWrite = pTable->numOfColumns; + } + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + if (pTable->info.type == TSDB_CHILD_TABLE) { + mgmtExtractTableName(pTable->superTableId, pWrite); + } + cols++; + + numOfRows++; + } + + pShow->numOfReads += numOfRows; + const int32_t NUM_OF_COLUMNS = 4; + + mgmtVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow); + + return numOfRows; +} + +void mgmtAlterChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable) { + int32_t code = TSDB_CODE_OPS_NOT_SUPPORT; + SCMAlterTableMsg *pAlter = pMsg->pCont;; + + if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { + code = mgmtModifyChildTableTagValueByName(pTable, pAlter->schema[0].name, pAlter->tagVal); + } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { + code = mgmtAddNormalTableColumn(pTable, pAlter->schema, 1); + } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { + code = mgmtDropNormalTableColumnByName(pTable, pAlter->schema[0].name); + } else { } - mTrace("db:%s, all child tables:%d is dropped", pDropDb->name, numOfTables); + mgmtSendSimpleResp(pMsg->thandle, code); } \ No newline at end of file diff --git a/src/mnode/src/mgmtDClient.c b/src/mnode/src/mgmtDClient.c index da11ad20811734d158aeaa12733ddae178402273..cbdcb8ff5dec6fae0684f8257a5379058622618e 100644 --- a/src/mnode/src/mgmtDClient.c +++ b/src/mnode/src/mgmtDClient.c @@ -42,7 +42,7 @@ int32_t mgmtInitDClient() { rpcInit.label = "MND-DC"; rpcInit.numOfThreads = 1; rpcInit.cfp = mgmtProcessRspFromDnode; - rpcInit.sessions = tsMaxDnodes * 5; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = "mgmtDClient"; diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index 177a45764ccd773a3ceb549fd0dccb624c4257c3..b1d01b91f1f147b600fc70bd696be8e986c3eadf 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -45,7 +45,7 @@ int32_t mgmtInitDServer() { rpcInit.label = "MND-DS"; rpcInit.numOfThreads = 1; rpcInit.cfp = mgmtProcessMsgFromDnode; - rpcInit.sessions = tsMaxDnodes * 5; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = mgmtDServerRetrieveAuth; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 88934599dc237b6451d8631b36db5583c37e819e..8c8173f781e20803910676b629cd5709b3a070a4 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -15,99 +15,124 @@ #define _DEFAULT_SOURCE #include "os.h" - -#include "mgmtDb.h" +#include "taoserror.h" +#include "tstatus.h" +#include "tutil.h" +#include "name.h" +#include "mnode.h" #include "mgmtAcct.h" #include "mgmtBalance.h" #include "mgmtChildTable.h" +#include "mgmtDb.h" #include "mgmtDnode.h" #include "mgmtGrant.h" -#include "mgmtMnode.h" -#include "mgmtNormalTable.h" #include "mgmtShell.h" +#include "mgmtMnode.h" +#include "mgmtChildTable.h" +#include "mgmtSdb.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" -#include "mnode.h" - -#include "taoserror.h" -#include "tstatus.h" -#include "tutil.h" -#include "name.h" static void *tsDbSdb = NULL; static int32_t tsDbUpdateSize; static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); static void mgmtDropDb(void *handle, void *tmrId); -static void mgmtSetDbDirty(SDbObj *pDb); - +static int32_t mgmtSetDbDirty(SDbObj *pDb); static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg); static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg); -static void *(*mgmtDbActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - -static void mgmtDbActionInit() { - mgmtDbActionFp[SDB_TYPE_INSERT] = mgmtDbActionInsert; - mgmtDbActionFp[SDB_TYPE_DELETE] = mgmtDbActionDelete; - mgmtDbActionFp[SDB_TYPE_UPDATE] = mgmtDbActionUpdate; - mgmtDbActionFp[SDB_TYPE_ENCODE] = mgmtDbActionEncode; - mgmtDbActionFp[SDB_TYPE_DECODE] = mgmtDbActionDecode; - mgmtDbActionFp[SDB_TYPE_RESET] = mgmtDbActionReset; - mgmtDbActionFp[SDB_TYPE_DESTROY] = mgmtDbActionDestroy; +static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) { + tfree(pOper->pObj); + return TSDB_CODE_SUCCESS; } -static void *mgmtDbAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtDbActionFp[(uint8_t)action] != NULL) { - return (*(mgmtDbActionFp[(uint8_t)action]))(row, str, size, ssize); +static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + + pDb->pHead = NULL; + pDb->pTail = NULL; + pDb->prev = NULL; + pDb->next = NULL; + pDb->numOfVgroups = 0; + pDb->numOfTables = 0; + pDb->numOfSuperTables = 0; + + if (pAcct != NULL) { + mgmtAddDbIntoAcct(pAcct, pDb); } - return NULL; + else { + mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); + return TSDB_CODE_INVALID_ACCT; + } + + return TSDB_CODE_SUCCESS; } -int32_t mgmtInitDbs() { - void * pNode = NULL; - SDbObj * pDb = NULL; - SAcctObj *pAcct = NULL; +static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtDbActionInit(); + mgmtRemoveDbFromAcct(pAcct, pDb); + mgmtDropAllChildTables(pDb); + mgmtDropAllSuperTables(pDb); + mgmtDropAllVgroups(pDb); + + return TSDB_CODE_SUCCESS; +} - SDbObj tObj; - tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; +static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; +} - tsDbSdb = sdbOpenTable(tsMaxDbs, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction); - if (tsDbSdb == NULL) { - mError("failed to init db data"); +static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + + if (pOper->maxRowSize < tsDbUpdateSize) { return -1; + } else { + memcpy(pOper->rowData, pDb, tsDbUpdateSize); + pOper->rowSize = tsDbUpdateSize; + return TSDB_CODE_SUCCESS; } +} - while (1) { - pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb); - if (pDb == NULL) break; +static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { + SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj)); + if (pDb == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; + + memcpy(pDb, pOper->rowData, tsDbUpdateSize); + pOper->pObj = pDb; + return TSDB_CODE_SUCCESS; +} - pDb->pHead = NULL; - pDb->pTail = NULL; - pDb->prev = NULL; - pDb->next = NULL; - pDb->numOfTables = 0; - pDb->numOfVgroups = 0; - pDb->numOfSuperTables = 0; - pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct != NULL) - mgmtAddDbIntoAcct(pAcct, pDb); - else { - mError("db:%s acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); - } +int32_t mgmtInitDbs() { + SDbObj tObj; + tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "dbs", + .hashSessions = TSDB_MAX_DBS, + .maxRowSize = tsDbUpdateSize, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtDbActionInsert, + .deleteFp = mgmtDbActionDelete, + .updateFp = mgmtDbActionUpdate, + .encodeFp = mgmtDbActionEncode, + .decodeFp = mgmtDbActionDecode, + .destroyFp = mgmtDbActionDestroy, + }; + + tsDbSdb = sdbOpenTable(&tableDesc); + if (tsDbSdb == NULL) { + mError("failed to init db data"); + return -1; } mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_DB, mgmtProcessCreateDbMsg); @@ -210,14 +235,14 @@ static int32_t mgmtCheckDBParams(SCMCreateDbMsg *pCreate) { static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { // assign default parameters - if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; // - if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; // - if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; // - if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; // - if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; // - if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; // - if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; // - if (pCreate->compression < 0) pCreate->compression = tsCompression; // + if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; + if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; + if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; + if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; + if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; + if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; + if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; + if (pCreate->compression < 0) pCreate->compression = tsCompression; if (pCreate->commitLog < 0) pCreate->commitLog = tsCommitLog; if (pCreate->replications < 0) pCreate->replications = tsReplications; // if (pCreate->rowsInFileBlock < 0) pCreate->rowsInFileBlock = tsRowsInFileBlock; // @@ -248,18 +273,10 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { pCreate->blocksPerTable = TSDB_MIN_AVG_BLOCKS; } - pCreate->maxSessions++; - return TSDB_CODE_SUCCESS; } static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { - int32_t numOfDbs = sdbGetNumOfRows(tsDbSdb); - if (numOfDbs >= tsMaxDbs) { - mWarn("numOfDbs:%d, exceed tsMaxDbs:%d", numOfDbs, tsMaxDbs); - return TSDB_CODE_TOO_MANY_DATABASES; - } - int32_t code = mgmtCheckDbLimit(pAcct); if (code != 0) { return code; @@ -287,9 +304,17 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { pDb->createdTime = taosGetTimestampMs(); pDb->cfg = *pCreate; - if (sdbInsertRow(tsDbSdb, pDb, 0) < 0) { - code = TSDB_CODE_SDB_ERROR; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj) + }; + + code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { tfree(pDb); + code = TSDB_CODE_SDB_ERROR; } return code; @@ -303,73 +328,8 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) { return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb)); } -static int32_t mgmtAlterDb(SAcctObj *pAcct, SCMAlterDbMsg *pAlter) { - return 0; -// int32_t code = TSDB_CODE_SUCCESS; -// -// SDbObj *pDb = (SDbObj *) sdbGetRow(tsDbSdb, pAlter->db); -// if (pDb == NULL) { -// mTrace("db:%s is not exist", pAlter->db); -// return TSDB_CODE_INVALID_DB; -// } -// -// int32_t oldReplicaNum = pDb->cfg.replications; -// if (pAlter->daysToKeep > 0) { -// mTrace("db:%s daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, pAlter->daysToKeep); -// pDb->cfg.daysToKeep = pAlter->daysToKeep; -// } else if (pAlter->replications > 0) { -// mTrace("db:%s replica:%d change to %d", pDb->name, pDb->cfg.replications, pAlter->replications); -// if (pAlter->replications < TSDB_REPLICA_MIN_NUM || pAlter->replications > TSDB_REPLICA_MAX_NUM) { -// mError("invalid db option replica: %d valid range: %d--%d", pAlter->replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); -// return TSDB_CODE_INVALID_OPTION; -// } -// pDb->cfg.replications = pAlter->replications; -// } else if (pAlter->maxSessions > 0) { -// mTrace("db:%s tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, pAlter->maxSessions); -// if (pAlter->maxSessions < TSDB_MIN_TABLES_PER_VNODE || pAlter->maxSessions > TSDB_MAX_TABLES_PER_VNODE) { -// mError("invalid db option tables: %d valid range: %d--%d", pAlter->maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); -// return TSDB_CODE_INVALID_OPTION; -// } -// if (pAlter->maxSessions < pDb->cfg.maxSessions) { -// mError("invalid db option tables: %d should larger than original:%d", pAlter->maxSessions, pDb->cfg.maxSessions); -// return TSDB_CODE_INVALID_OPTION; -// } -// return TSDB_CODE_INVALID_OPTION; -// //The modification of tables needs to rewrite the head file, so disable this option -// //pDb->cfg.maxSessions = pAlter->maxSessions; -// } else { -// mError("db:%s alter msg, replica:%d, keep:%d, tables:%d, origin replica:%d keep:%d", pDb->name, -// pAlter->replications, pAlter->maxSessions, pAlter->daysToKeep, -// pDb->cfg.replications, pDb->cfg.daysToKeep); -// return TSDB_CODE_INVALID_OPTION; -// } -// -// if (sdbUpdateRow(tsDbSdb, pDb, tsDbUpdateSize, 1) < 0) { -// return TSDB_CODE_SDB_ERROR; -// } -// -// SVgObj *pVgroup = pDb->pHead; -// while (pVgroup != NULL) { -// balanceUpdateVgroupState(pVgroup, TSDB_VG_LB_STATUS_UPDATE, 0); -// if (oldReplicaNum < pDb->cfg.replications) { -// if (!balanceAddVnode(pVgroup, NULL, NULL)) { -// mWarn("db:%s vgroup:%d not enough dnode to add vnode", pAlter->db, pVgroup->vgId); -// code = TSDB_CODE_NO_ENOUGH_DNODES; -// } -// } -// if (pAlter->maxSessions > 0) { -// //rebuild meterList in mgmtVgroup.c -// mgmtUpdateVgroup(pVgroup); -// } -//// mgmtSendCreateVnodeMsg(pVgroup); -// pVgroup = pVgroup->next; -// } -// mgmtStartBalanceTimer(10); -// -// return code; -} - -int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtAddVgroupIntoDb(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; pVgroup->next = pDb->pHead; pVgroup->prev = NULL; @@ -378,11 +338,10 @@ int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { pDb->pHead = pVgroup; pDb->numOfVgroups++; - - return 0; } -int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; pVgroup->next = NULL; pVgroup->prev = pDb->pTail; @@ -391,32 +350,25 @@ int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) { pDb->pTail = pVgroup; pDb->numOfVgroups++; - - return 0; } -int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; if (pVgroup->prev) pVgroup->prev->next = pVgroup->next; if (pVgroup->next) pVgroup->next->prev = pVgroup->prev; if (pVgroup->prev == NULL) pDb->pHead = pVgroup->next; if (pVgroup->next == NULL) pDb->pTail = pVgroup->prev; pDb->numOfVgroups--; - - return 0; } -int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtAddVgroupIntoDbTail(pDb, pVgroup); - - return 0; +void mgmtMoveVgroupToTail(SVgObj *pVgroup) { + mgmtRemoveVgroupFromDb(pVgroup); + mgmtAddVgroupIntoDbTail(pVgroup); } -int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtAddVgroupIntoDb(pDb, pVgroup); - - return 0; +void mgmtMoveVgroupToHead(SVgObj *pVgroup) { + mgmtRemoveVgroupFromDb(pVgroup); + mgmtAddVgroupIntoDb(pVgroup); } void mgmtCleanUpDbs() { @@ -427,7 +379,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) int32_t cols = 0; SSchema *pSchema = pMeta->schema; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; pShow->bytes[cols] = TSDB_DB_NAME_LEN; @@ -438,7 +390,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -566,7 +518,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) return 0; } -char *mgmtGetDbStr(char *src) { +static char *mgmtGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); return ++pos; @@ -577,7 +529,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * SDbObj *pDb = NULL; char * pWrite; int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; while (numOfRows < rows) { @@ -636,7 +588,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * if (strcmp(pUser->user, "root") == 0) { #endif pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxSessions - 1; // table num can be created should minus 1 + *(int32_t *)pWrite = pDb->cfg.maxSessions; // table num can be created should minus 1 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -649,9 +601,9 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; #ifdef _TD_ARM_32_ - *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); + *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); #else - *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); + *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); #endif cols++; @@ -691,68 +643,6 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * return numOfRows; } -void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - - pDb->pHead = NULL; - pDb->pTail = NULL; - pDb->numOfVgroups = 0; - pDb->numOfTables = 0; - mgmtAddDbIntoAcct(pAcct, pDb); - - return NULL; -} - -void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtRemoveDbFromAcct(pAcct, pDb); - - mgmtDropAllNormalTables(pDb); - mgmtDropAllChildTables(pDb); - mgmtDropAllSuperTables(pDb); - - return NULL; -} - -void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtDbActionReset(row, str, size, ssize); -} - -void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - if (size < tsDbUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pDb, tsDbUpdateSize); - *ssize = tsDbUpdateSize; - } - - return NULL; -} -void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) malloc(sizeof(SDbObj)); - if (pDb == NULL) return NULL; - memset(pDb, 0, sizeof(SDbObj)); - - memcpy(pDb, str, tsDbUpdateSize); - - return (void *)pDb; -} - -void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - memcpy(pDb, str, tsDbUpdateSize); - - return NULL; -} - -void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - tfree(row); - return NULL; -} - void mgmtAddSuperTableIntoDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfSuperTables, 1); } @@ -768,8 +658,23 @@ void mgmtRemoveTableFromDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfTables, -1); } -static void mgmtSetDbDirty(SDbObj *pDb) { +static int32_t mgmtSetDbDirty(SDbObj *pDb) { + if (pDb->dirty) return TSDB_CODE_SUCCESS; + pDb->dirty = true; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = tsDbUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_SDB_ERROR; + } + + return code; } static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { @@ -794,32 +699,121 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { } else { code = mgmtCreateDb(pMsg->pUser->pAcct, pCreate); if (code == TSDB_CODE_SUCCESS) { - mLPrint("DB:%s is created by %s", pCreate->db, pMsg->pUser->user); + mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user); } } mgmtSendSimpleResp(pMsg->thandle, code); } +static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { + SDbCfg newCfg = pDb->cfg; + int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t maxSessions = htonl(pAlter->maxSessions); + int8_t replications = pAlter->replications; + + terrno = TSDB_CODE_SUCCESS; + + if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) { + mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep); + newCfg.daysToKeep = daysToKeep; + } else if (replications > 0 && replications != pDb->cfg.replications) { + mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); + if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) { + mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + terrno = TSDB_CODE_INVALID_OPTION; + } + newCfg.replications = replications; + } else if (maxSessions > 0 && maxSessions != pDb->cfg.maxSessions) { + mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, maxSessions); + if (maxSessions < TSDB_MIN_TABLES_PER_VNODE || maxSessions > TSDB_MAX_TABLES_PER_VNODE) { + mError("invalid db option tables: %d valid range: %d--%d", maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + terrno = TSDB_CODE_INVALID_OPTION; + } + if (maxSessions < pDb->cfg.maxSessions) { + mError("invalid db option tables: %d should larger than original:%d", maxSessions, pDb->cfg.maxSessions); + terrno = TSDB_CODE_INVALID_OPTION; + } + newCfg.maxSessions = maxSessions; + } else { + } + + return newCfg; +} + +static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { + SDbCfg newCfg = mgmtGetAlterDbOption(pDb, pAlter); + if (terrno != TSDB_CODE_SUCCESS) { + return terrno; + } + + if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { + pDb->cfg = newCfg; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = tsDbUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_SDB_ERROR; + } + } + + return TSDB_CODE_SUCCESS; +} + static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) { if (mgmtCheckRedirect(pMsg->thandle)) return; SCMAlterDbMsg *pAlter = pMsg->pCont; - pAlter->daysPerFile = htonl(pAlter->daysPerFile); - pAlter->daysToKeep = htonl(pAlter->daysToKeep); - pAlter->maxSessions = htonl(pAlter->maxSessions) + 1; + mTrace("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->thandle); + + if (mgmtCheckExpired()) { + mError("db:%s, failed to alter, grant expired", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); + return; + } - int32_t code; if (!pMsg->pUser->writeAuth) { - code = TSDB_CODE_NO_RIGHTS; - } else { - code = mgmtAlterDb(pMsg->pUser->pAcct, pAlter); - if (code == TSDB_CODE_SUCCESS) { - mLPrint("DB:%s is altered by %s", pAlter->db, pMsg->pUser->user); - } + mError("db:%s, failed to alter, no rights", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); + return; } - mgmtSendSimpleResp(pMsg->thandle, code); + SDbObj *pDb = mgmtGetDb(pAlter->db); + if (pDb == NULL) { + mError("db:%s, failed to alter, invalid db", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB); + return; + } + + int32_t code = mgmtAlterDb(pDb, pAlter); + if (code != TSDB_CODE_SUCCESS) { + mError("db:%s, failed to alter, invalid db option", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, code); + } + + SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); + memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); + pMsg->pCont = NULL; + + SVgObj *pVgroup = pDb->pHead; + if (pVgroup != NULL) { + mPrint("vgroup:%d, will be altered", pVgroup->vgId); + newMsg->ahandle = pVgroup; + newMsg->expected = pVgroup->numOfVnodes; + mgmtAlterVgroup(pVgroup, newMsg); + return; + } + + mTrace("db:%s, all vgroups is altered", pDb->name); + + mgmtSendSimpleResp(newMsg->thandle, TSDB_CODE_SUCCESS); + rpcFreeCont(newMsg->pCont); + free(newMsg); } static void mgmtDropDb(void *handle, void *tmrId) { @@ -827,7 +821,12 @@ static void mgmtDropDb(void *handle, void *tmrId) { SDbObj *pDb = newMsg->ahandle; mPrint("db:%s, drop db from sdb", pDb->name); - int32_t code = sdbDeleteRow(tsDbSdb, pDb); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb + }; + int32_t code = sdbDeleteRow(&oper); if (code != 0) { code = TSDB_CODE_SDB_ERROR; } @@ -874,7 +873,12 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) { return; } - mgmtSetDbDirty(pDb); + int32_t code = mgmtSetDbDirty(pDb); + if (code != TSDB_CODE_SUCCESS) { + mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code)); + mgmtSendSimpleResp(pMsg->thandle, code); + return; + } SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index cf57b43918fb5fd89665d6aee770007a0768a1d5..291cbe8a48a37e1ddf02b292a6746d4bf45a0917 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -76,7 +76,7 @@ bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; @@ -169,7 +169,7 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; @@ -256,7 +256,7 @@ static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, vo static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; @@ -547,9 +547,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return ; } - uint32_t lastPrivateIp = pDnode->privateIp; - uint32_t lastPublicIp = pDnode->publicIp; - pDnode->privateIp = htonl(pStatus->privateIp); pDnode->publicIp = htonl(pStatus->publicIp); pDnode->lastReboot = htonl(pStatus->lastReboot); @@ -566,11 +563,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { mgmtSetDnodeMaxVnodes(pDnode); } - if (lastPrivateIp != pDnode->privateIp || lastPublicIp != pDnode->publicIp) { - mgmtUpdateVgroupIp(pDnode); - //mgmtUpdateMnodeIp(); - } - int32_t openVnodes = htons(pStatus->openVnodes); for (int32_t j = 0; j < openVnodes; ++j) { pDnode->vload[j].vgId = htonl(pStatus->load[j].vgId); @@ -599,7 +591,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return; } - mgmtGetMnodeIpList(&pRsp->ipList); + mgmtGetMnodePrivateIpList(&pRsp->ipList); pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus); diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index 66200e5a1407aa59e1e2a76f736c23eb0f82a7b3..ab0b99fa586512e64d1550b29ab77e1b4cbe4332 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -25,6 +25,8 @@ #include "mgmtDClient.h" #include "mgmtDnode.h" #include "mgmtDServer.h" +#include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtVgroup.h" #include "mgmtUser.h" #include "mgmtTable.h" @@ -65,7 +67,7 @@ int32_t mgmtStartSystem() { return 0; } - tsMgmtTmr = taosTmrInit((tsMaxDnodes + tsMaxShellConns) * 3, 200, 3600000, "MND"); + tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND"); if (tsMgmtTmr == NULL) { mError("failed to init timer"); return -1; @@ -109,8 +111,8 @@ int32_t mgmtStartSystem() { return -1; } - if (sdbInitPeers(tsMnodeDir) < 0) { - mError("failed to init peers"); + if (mgmtInitMnodes() < 0) { + mError("failed to init mnodes"); return -1; } @@ -125,7 +127,7 @@ int32_t mgmtStartSystem() { void mgmtStopSystem() { - if (sdbMaster) { + if (mgmtIsMaster()) { mTrace("it is a master mgmt node, it could not be stopped"); return; } @@ -136,7 +138,7 @@ void mgmtStopSystem() { void mgmtCleanUpSystem() { mPrint("starting to clean up mgmt"); - sdbCleanUpPeers(); + mgmtCleanupMnodes(); mgmtCleanupBalance(); mgmtCleanUpShell(); mgmtCleanupDClient(); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index f7e6e3f8cb4c984917a30ff7000e7d4373619c3b..71ce50eccbbc706e66e35387da63152433bcae5c 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -17,48 +17,93 @@ #include "os.h" #include "trpc.h" #include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtUser.h" -int32_t (*mgmtAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; -int32_t (*mgmtRemoveMnodeFp)(uint32_t privateIp) = NULL; -int32_t (*mgmtGetMnodesNumFp)() = NULL; -void * (*mgmtGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; +int32_t (*mpeerAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; +int32_t (*mpeerRemoveMnodeFp)(uint32_t privateIp) = NULL; +int32_t (*mpeerGetMnodesNumFp)() = NULL; +void * (*mpeerGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; +int32_t (*mpeerInitMnodesFp)() = NULL; +void (*mpeerCleanUpMnodesFp)() = NULL; static SMnodeObj tsMnodeObj = {0}; +static bool tsMnodeIsMaster = false; +static bool tsMnodeIsServing = false; static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static char *mgmtMnodeStatusStr[] = { + "offline", + "unsynced", + "syncing", + "serving", + "null" +}; + +static char *mgmtMnodeRoleStr[] = { + "unauthed", + "undecided", + "master", + "slave", + "null" +}; + +int32_t mgmtInitMnodes() { + if (mpeerInitMnodesFp) { + return (*mpeerInitMnodesFp)(); + } else { + tsMnodeIsServing = true; + tsMnodeIsMaster = true; + return 0; + } +} + +void mgmtCleanupMnodes() { + if (mpeerCleanUpMnodesFp) { + (*mpeerCleanUpMnodesFp)(); + } +} + +bool mgmtInServerStatus() { + return tsMnodeIsServing; +} + +bool mgmtIsMaster() { + return tsMnodeIsMaster; +} + bool mgmtCheckRedirect(void *handle) { return false; } int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp) { - if (mgmtAddMnodeFp) { - return (*mgmtAddMnodeFp)(privateIp, publicIp); + if (mpeerAddMnodeFp) { + return (*mpeerAddMnodeFp)(privateIp, publicIp); } else { return 0; } } int32_t mgmtRemoveMnode(uint32_t privateIp) { - if (mgmtRemoveMnodeFp) { - return (*mgmtRemoveMnodeFp)(privateIp); + if (mpeerRemoveMnodeFp) { + return (*mpeerRemoveMnodeFp)(privateIp); } else { return 0; } } static int32_t mgmtGetMnodesNum() { - if (mgmtGetMnodesNumFp) { - return (*mgmtGetMnodesNumFp)(); + if (mpeerGetMnodesNumFp) { + return (*mpeerGetMnodesNumFp)(); } else { return 1; } } static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) { - if (mgmtGetNextMnodeFp) { - return (*mgmtGetNextMnodeFp)(pShow, pMnode); + if (mpeerGetNextMnodeFp) { + return (*mpeerGetNextMnodeFp)(pShow, pMnode); } else { if (*pMnode == NULL) { *pMnode = &tsMnodeObj; @@ -73,7 +118,7 @@ static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) { static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; @@ -148,11 +193,11 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbStatusStr[(uint8_t)pMnode->status]); + strcpy(pWrite, mgmtMnodeStatusStr[pMnode->status]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbRoleStr[(uint8_t)pMnode->role]); + strcpy(pWrite, mgmtMnodeRoleStr[pMnode->role]); cols++; tinet_ntoa(ipstr, pMnode->publicIp); @@ -167,7 +212,14 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi return numOfRows; } -void mgmtGetMnodeIpList(SRpcIpSet *ipSet) { +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) { + ipSet->inUse = 0; + ipSet->port = htons(tsMnodeDnodePort); + ipSet->numOfIps = 1; + ipSet->ip[0] = htonl(inet_addr(tsMasterIp)); +} + +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c deleted file mode 100644 index f076b69f36323f08c0e69195d6d1ab08b4ac4150..0000000000000000000000000000000000000000 --- a/src/mnode/src/mgmtNormalTable.c +++ /dev/null @@ -1,575 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosmsg.h" -#include "tscompression.h" -#include "tskiplist.h" -#include "ttime.h" -#include "tstatus.h" -#include "tutil.h" -#include "mnode.h" -#include "mgmtAcct.h" -#include "mgmtDb.h" -#include "mgmtDClient.h" -#include "mgmtGrant.h" -#include "mgmtNormalTable.h" -#include "mgmtSuperTable.h" -#include "mgmtTable.h" -#include "mgmtVgroup.h" - -void *tsNormalTableSdb; -int32_t tsNormalTableUpdateSize; -void *(*mgmtNormalTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); - -void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - -static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { - free(pTable->schema); - free(pTable->sql); - free(pTable); -} - -static void mgmtNormalTableActionInit() { - mgmtNormalTableActionFp[SDB_TYPE_INSERT] = mgmtNormalTableActionInsert; - mgmtNormalTableActionFp[SDB_TYPE_DELETE] = mgmtNormalTableActionDelete; - mgmtNormalTableActionFp[SDB_TYPE_UPDATE] = mgmtNormalTableActionUpdate; - mgmtNormalTableActionFp[SDB_TYPE_ENCODE] = mgmtNormalTableActionEncode; - mgmtNormalTableActionFp[SDB_TYPE_DECODE] = mgmtNormalTableActionDecode; - mgmtNormalTableActionFp[SDB_TYPE_RESET] = mgmtNormalTableActionReset; - mgmtNormalTableActionFp[SDB_TYPE_DESTROY] = mgmtNormalTableActionDestroy; -} - -void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - memcpy(pTable, str, tsNormalTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns) + pTable->sqlLen; - pTable->schema = realloc(pTable->schema, schemaSize); - pTable->sql = (char*)pTable->schema + sizeof(SSchema) * (pTable->numOfColumns); - memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); - - return NULL; -} - -void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *)row; - mgmtDestroyNormalTable(pTable); - return NULL; -} - -void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - mError("id:%s not in vgroup:%d", pTable->tableId, pTable->vgId); - return NULL; - } - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb == NULL) { - mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); - return NULL; - } - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct == NULL) { - mError("account not exists"); - return NULL; - } - - if (!sdbMaster) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid != pTable->sid) { - mError("sid:%d is not matched from the master:%d", sid, pTable->sid); - return NULL; - } - } - - mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); - mgmtAddTableIntoDb(pDb); - mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); - } - - return NULL; -} - -void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - if (pTable->vgId == 0) { - return NULL; - } - - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - return NULL; - } - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb == NULL) { - mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); - return NULL; - } - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct == NULL) { - mError("account not exists"); - return NULL; - } - - mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1); - mgmtRemoveTableFromDb(pDb); - mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - - if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pDb, pVgroup); - } - - return NULL; -} - -void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtNormalTableActionReset(row, str, size, NULL); -} - -void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - assert(row != NULL && str != NULL); - - int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (size < tsNormalTableUpdateSize + schemaSize + 1) { - *ssize = -1; - return NULL; - } - - memcpy(str, pTable, tsNormalTableUpdateSize); - memcpy(str + tsNormalTableUpdateSize, pTable->schema, schemaSize); - memcpy(str + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - *ssize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; - - return NULL; -} - -void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); - - SNormalTableObj *pTable = (SNormalTableObj *)malloc(sizeof(SNormalTableObj)); - if (pTable == NULL) { - return NULL; - } - memset(pTable, 0, sizeof(SNormalTableObj)); - - if (size < tsNormalTableUpdateSize) { - mgmtDestroyNormalTable(pTable); - return NULL; - } - memcpy(pTable, str, tsNormalTableUpdateSize); - - int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - pTable->schema = (SSchema *)malloc(schemaSize); - if (pTable->schema == NULL) { - mgmtDestroyNormalTable(pTable); - return NULL; - } - - memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); - - pTable->sql = (char *)malloc(pTable->sqlLen); - if (pTable->sql == NULL) { - mgmtDestroyNormalTable(pTable); - return NULL; - } - memcpy(pTable->sql, str + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); - return (void *)pTable; -} - -void *mgmtNormalTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtNormalTableActionFp[(uint8_t)action] != NULL) { - return (*(mgmtNormalTableActionFp[(uint8_t)action]))(row, str, size, ssize); - } - return NULL; -} - -int32_t mgmtInitNormalTables() { - void *pNode = NULL; - void *pLastNode = NULL; - SNormalTableObj *pTable = NULL; - - mgmtNormalTableActionInit(); - SNormalTableObj tObj; - tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - tsNormalTableSdb = sdbOpenTable(tsMaxTables, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, - "ntables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtNormalTableAction); - if (tsNormalTableSdb == NULL) { - mError("failed to init ntables data"); - return -1; - } - - while (1) { - pLastNode = pNode; - pNode = sdbFetchRow(tsNormalTableSdb, pNode, (void **)&pTable); - if (pTable == NULL) break; - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("ntable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsNormalTableSdb, pTable); - pNode = pLastNode; - continue; - } - - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - mError("ntable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); - pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); - pNode = pLastNode; - continue; - } - - if (strcmp(pVgroup->dbName, pDb->name) != 0) { - mError("ntable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", - pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); - pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); - pNode = pLastNode; - continue; - } - - if (pVgroup->tableList == NULL) { - mError("ntable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); - pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); - pNode = pLastNode; - continue; - } - - mgmtAddTableIntoVgroup(pVgroup, (STableInfo *)pTable); - //pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1); - - pTable->sql = (char *)pTable->schema + sizeof(SSchema) * pTable->numOfColumns; - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); - } - - mTrace("ntables is initialized"); - return 0; -} - -void mgmtCleanUpNormalTables() { - sdbCloseTable(tsNormalTableSdb); -} - -void *mgmtBuildCreateNormalTableMsg(SNormalTableObj *pTable) { - int32_t totalCols = pTable->numOfColumns; - int32_t contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + pTable->sqlLen; - - SMDCreateTableMsg *pCreate = rpcMallocCont(contLen); - if (pCreate == NULL) { - terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; - return NULL; - } - - memcpy(pCreate->tableId, pTable->tableId, TSDB_TABLE_ID_LEN + 1); - pCreate->contLen = htonl(contLen); - pCreate->vgId = htonl(pTable->vgId); - pCreate->tableType = pTable->type; - pCreate->numOfColumns = htons(pTable->numOfColumns); - pCreate->numOfTags = 0; - pCreate->sid = htonl(pTable->sid); - pCreate->sversion = htonl(pTable->sversion); - pCreate->tagDataLen = 0; - pCreate->sqlDataLen = htonl(pTable->sqlLen); - pCreate->uid = htobe64(pTable->uid); - pCreate->superTableUid = 0; - pCreate->createdTime = htobe64(pTable->createdTime); - - SSchema *pSchema = (SSchema *) pCreate->data; - memcpy(pSchema, pTable->schema, totalCols * sizeof(SSchema)); - for (int32_t col = 0; col < totalCols; ++col) { - pSchema->bytes = htons(pSchema->bytes); - pSchema->colId = htons(pSchema->colId); - pSchema++; - } - - memcpy(pCreate + sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema), pTable->sql, pTable->sqlLen); - return pCreate; -} - -void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid) { - int32_t numOfTables = sdbGetNumOfRows(tsNormalTableSdb); - if (numOfTables >= TSDB_MAX_NORMAL_TABLES) { - mError("table:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_NORMAL_TABLES); - terrno = TSDB_CODE_TOO_MANY_TABLES; - return NULL; - } - - SNormalTableObj *pTable = (SNormalTableObj *) calloc(sizeof(SNormalTableObj), 1); - if (pTable == NULL) { - mError("table:%s, failed to alloc memory", pCreate->tableId); - terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; - return NULL; - } - - strcpy(pTable->tableId, pCreate->tableId); - pTable->type = TSDB_NORMAL_TABLE; - pTable->vgId = pVgroup->vgId; - pTable->createdTime = taosGetTimestampMs(); - pTable->uid = (((uint64_t) pTable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); - pTable->sid = sid; - pTable->sversion = 0; - pTable->numOfColumns = htons(pCreate->numOfColumns); - pTable->sqlLen = htons(pCreate->sqlLen); - - int32_t numOfCols = pTable->numOfColumns; - int32_t schemaSize = numOfCols * sizeof(SSchema); - pTable->schema = (SSchema *) calloc(1, schemaSize); - if (pTable->schema == NULL) { - free(pTable); - terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; - return NULL; - } - memcpy(pTable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); - - pTable->nextColId = 0; - for (int32_t col = 0; col < numOfCols; col++) { - SSchema *tschema = pTable->schema; - tschema[col].colId = pTable->nextColId++; - tschema[col].bytes = htons(tschema[col].bytes); - } - - if (pTable->sqlLen != 0) { - pTable->type = TSDB_STREAM_TABLE; - pTable->sql = calloc(1, pTable->sqlLen); - if (pTable->sql == NULL) { - free(pTable); - terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; - return NULL; - } - memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); - pTable->sql[pTable->sqlLen - 1] = 0; - mTrace("table:%s, stream sql len:%d sql:%s", pTable->tableId, pTable->sqlLen, pTable->sql); - } - - if (sdbInsertRow(tsNormalTableSdb, pTable, 0) < 0) { - mError("table:%s, update sdb error", pTable->tableId); - free(pTable); - terrno = TSDB_CODE_SDB_ERROR; - return NULL; - } - - mTrace("table:%s, create ntable in vgroup, uid:%" PRIu64 , pTable->tableId, pTable->uid); - return pTable; -} - -int32_t mgmtDropNormalTable(SQueuedMsg *newMsg, SNormalTableObj *pTable) { - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - mError("table:%s, failed to drop normal table, vgroup not exist", pTable->tableId); - return TSDB_CODE_OTHERS; - } - - SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg)); - if (pDrop == NULL) { - mError("table:%s, failed to drop normal table, no enough memory", pTable->tableId); - return TSDB_CODE_SERV_OUT_OF_MEMORY; - } - - strcpy(pDrop->tableId, pTable->tableId); - pDrop->contLen = htonl(sizeof(SMDDropTableMsg)); - pDrop->vgId = htonl(pVgroup->vgId); - pDrop->sid = htonl(pTable->sid); - pDrop->uid = htobe64(pTable->uid); - - SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); - mTrace("table:%s, send drop table msg", pDrop->tableId); - SRpcMsg rpcMsg = { - .handle = newMsg, - .pCont = pDrop, - .contLen = sizeof(SMDDropTableMsg), - .code = 0, - .msgType = TSDB_MSG_TYPE_MD_DROP_TABLE - }; - - newMsg->ahandle = pTable; - mgmtSendMsgToDnode(&ipSet, &rpcMsg); - return TSDB_CODE_SUCCESS; -} - -void* mgmtGetNormalTable(char *tableId) { - return sdbGetRow(tsNormalTableSdb, tableId); -} - -static int32_t mgmtFindNormalTableColumnIndex(SNormalTableObj *pTable, char *colName) { - SSchema *schema = (SSchema *) pTable->schema; - for (int32_t i = 0; i < pTable->numOfColumns; i++) { - if (strcasecmp(schema[i].name, colName) == 0) { - return i; - } - } - - return -1; -} - -int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int32_t ncols) { - if (ncols <= 0) { - return TSDB_CODE_APP_ERROR; - } - - for (int32_t i = 0; i < ncols; i++) { - if (mgmtFindNormalTableColumnIndex(pTable, schema[i].name) > 0) { - return TSDB_CODE_APP_ERROR; - } - } - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("table: %s not belongs to any database", pTable->tableId); - return TSDB_CODE_APP_ERROR; - } - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct == NULL) { - mError("DB: %s not belongs to andy account", pDb->name); - return TSDB_CODE_APP_ERROR; - } - - int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - pTable->schema = realloc(pTable->schema, schemaSize + sizeof(SSchema) * ncols); - - memcpy(pTable->schema + schemaSize, schema, sizeof(SSchema) * ncols); - - SSchema *tschema = (SSchema *) (pTable->schema + sizeof(SSchema) * pTable->numOfColumns); - for (int32_t i = 0; i < ncols; i++) { - tschema[i].colId = pTable->nextColId++; - } - - pTable->numOfColumns += ncols; - pTable->sversion++; - pAcct->acctInfo.numOfTimeSeries += ncols; - - sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1); - return TSDB_CODE_SUCCESS; -} - -int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) { - int32_t col = mgmtFindNormalTableColumnIndex(pTable, colName); - if (col < 0) { - return TSDB_CODE_APP_ERROR; - } - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("table: %s not belongs to any database", pTable->tableId); - return TSDB_CODE_APP_ERROR; - } - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct == NULL) { - mError("DB: %s not belongs to any account", pDb->name); - return TSDB_CODE_APP_ERROR; - } - - memmove(pTable->schema + sizeof(SSchema) * col, pTable->schema + sizeof(SSchema) * (col + 1), - sizeof(SSchema) * (pTable->numOfColumns - col - 1)); - - pTable->numOfColumns--; - pTable->sversion++; - - pAcct->acctInfo.numOfTimeSeries--; - sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1); - - return TSDB_CODE_SUCCESS; -} - -static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SNormalTableObj *pTable) { - int32_t numOfCols = pTable->numOfColumns; - for (int32_t i = 0; i < numOfCols; ++i) { - strcpy(pSchema->name, pTable->schema[i].name); - pSchema->type = pTable->schema[i].type; - pSchema->bytes = htons(pTable->schema[i].bytes); - pSchema->colId = htons(pTable->schema[i].colId); - pSchema++; - } - - return numOfCols * sizeof(SSchema); -} - -int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { - pMeta->uid = htobe64(pTable->uid); - pMeta->sid = htonl(pTable->sid); - pMeta->vgId = htonl(pTable->vgId); - pMeta->sversion = htons(pTable->sversion); - pMeta->precision = pDb->cfg.precision; - pMeta->numOfTags = 0; - pMeta->numOfColumns = htons(pTable->numOfColumns); - pMeta->tableType = pTable->type; - pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable); - - strncpy(pMeta->tableId, pTable->tableId, tListLen(pTable->tableId)); - - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - return TSDB_CODE_INVALID_TABLE; - } - for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) { - if (usePublicIp) { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; - } else { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp; - } - - pMeta->vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); - } - pMeta->numOfVpeers = pVgroup->numOfVnodes; - - return TSDB_CODE_SUCCESS; -} - -void mgmtDropAllNormalTables(SDbObj *pDropDb) { - void *pNode = NULL; - void *pLastNode = NULL; - int32_t numOfTables = 0; - int32_t dbNameLen = strlen(pDropDb->name); - SNormalTableObj *pTable = NULL; - - while (1) { - pNode = sdbFetchRow(tsNormalTableSdb, pNode, (void **)&pTable); - if (pTable == NULL) break; - - if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsNormalTableSdb, pTable); - pNode = pLastNode; - numOfTables ++; - continue; - } - } - - mTrace("db:%s, all normal tables:%d is dropped", pDropDb->name, numOfTables); -} diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index b20e66dd874b6b589087b37e66f463e5d32aee89..44bd45faea5fd8d9414e559c2e3bb9c65cdbb8d3 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -678,7 +678,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; if (mgmtCheckRedirect(pMsg->thandle)) return; - SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle); + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; rpcSendResponse(&rpcRsp); @@ -702,7 +702,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; if (mgmtCheckRedirect(pMsg->thandle)) return; - SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle); + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; rpcSendResponse(&rpcRsp); @@ -726,7 +726,7 @@ void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; if (mgmtCheckRedirect(pMsg->thandle)) return; - SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle); + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; rpcSendResponse(&rpcRsp); @@ -762,3 +762,13 @@ int32_t mgmtInitProfile() { void mgmtCleanUpProfile() { } + +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) { + if (pMsg != NULL) { + if (pMsg->pCont != NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + free(pMsg); + } +} \ No newline at end of file diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c new file mode 100644 index 0000000000000000000000000000000000000000..0b1ac3be9706ea8c7342914a4fa87e06ca7cb94c --- /dev/null +++ b/src/mnode/src/mgmtSdb.c @@ -0,0 +1,767 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tchecksum.h" +#include "tglobalcfg.h" +#include "tlog.h" +#include "trpc.h" +#include "tutil.h" +#include "hashint.h" +#include "hashstr.h" +#include "mgmtSdb.h" + +#define abs(x) (((x) < 0) ? -(x) : (x)) +#define SDB_MAX_PEERS 4 +#define SDB_DELIMITER 0xFFF00F00 +#define SDB_ENDCOMMIT 0xAFFFAAAF + +typedef struct { + uint64_t swVersion; + int16_t sdbFileVersion; + char reserved[2]; + TSCKSUM checkSum; +} SSdbHeader; + +typedef struct _SSdbTable { + SSdbHeader header; + char tableName[TSDB_DB_NAME_LEN]; + char fileName[TSDB_FILENAME_LEN]; + ESdbKeyType keyType; + int32_t tableId; + int32_t hashSessions; + int32_t maxRowSize; + int32_t autoIndex; + int32_t fd; + int64_t numOfRows; + int64_t version; + int64_t fileSize; + void * iHandle; + int32_t (*insertFp)(SSdbOperDesc *pDesc); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*destroyFp)(SSdbOperDesc *pOper); + pthread_mutex_t mutex; +} SSdbTable; + +typedef struct { + int64_t version; + int64_t offset; + int32_t rowSize; + void * row; +} SRowMeta; + +typedef struct { + int32_t delimiter; + int32_t rowSize; + int64_t version; + char data[]; +} SRowHead; + +typedef enum { + SDB_FORWARD_TYPE_INSERT, + SDB_FORWARD_TYPE_DELETE, + SDB_FORWARD_TYPE_UPDATE +} ESdbForwardType; + +typedef struct { + ESdbForwardType type; + int32_t tableId; + int64_t version; + int32_t rowSize; + void * rowData; +} SForwardMsg; + +extern char version[]; +const int16_t sdbFileVersion = 2; +int32_t (*mpeerForwardRequestFp)(SForwardMsg *forwardMsg) = NULL; + +static SSdbTable *sdbTableList[10] = {0}; +static int32_t sdbNumOfTables = 0; +static uint64_t sdbVersion = 0; + +static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash}; +static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash}; +static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash}; +static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData}; +static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; +static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData}; + +uint64_t sdbGetVersion() { return sdbVersion; } +int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; } +int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } + +static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { + static char str[16]; + switch (pTable->keyType) { + case SDB_KEY_TYPE_STRING: + return (char *)row; + case SDB_KEY_TYPE_AUTO: + sprintf(str, "%d", *(int32_t *)row); + return str; + default: + return "unknown"; + } +} + +static int32_t sdbForwardDbReqToPeer(SForwardMsg *forwardMsg) { + if (mpeerForwardRequestFp) { + return mpeerForwardRequestFp(forwardMsg); + } else { + return 0; + } +} + +static void sdbFinishCommit(SSdbTable *pTable) { + uint32_t sdbEcommit = SDB_ENDCOMMIT; + off_t offset = lseek(pTable->fd, 0, SEEK_END); + assert(offset == pTable->fileSize); + twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); + pTable->fileSize += sizeof(sdbEcommit); +} + +static int32_t sdbOpenSdbFile(SSdbTable *pTable) { + struct stat fstat, ofstat; + uint64_t size; + char * dirc = NULL; + char * basec = NULL; + union { + char cversion[64]; + uint64_t iversion; + } swVersion; + + memcpy(swVersion.cversion, version, sizeof(uint64_t)); + + // check sdb.db and .sdb.db status + char fn[TSDB_FILENAME_LEN] = "\0"; + dirc = strdup(pTable->fileName); + basec = strdup(pTable->fileName); + sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); + tfree(dirc); + tfree(basec); + if (stat(fn, &ofstat) == 0) { // .sdb.db file exists + if (stat(pTable->fileName, &fstat) == 0) { + remove(fn); + } else { + remove(pTable->fileName); + rename(fn, pTable->fileName); + } + } + + pTable->fd = open(pTable->fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + if (pTable->fd < 0) { + sdbError("table:%s, failed to open file:%s", pTable->tableName, pTable->fileName); + return -1; + } + + pTable->fileSize = 0; + stat(pTable->fileName, &fstat); + size = sizeof(pTable->header); + + if (fstat.st_size == 0) { + pTable->header.swVersion = swVersion.iversion; + pTable->header.sdbFileVersion = sdbFileVersion; + if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) { + sdbError("table:%s, failed to get file header checksum, file:%s", pTable->tableName, pTable->fileName); + tclose(pTable->fd); + return -1; + } + twrite(pTable->fd, &(pTable->header), size); + pTable->fileSize += size; + sdbFinishCommit(pTable); + } else { + uint32_t sdbEcommit = 0; + off_t offset = lseek(pTable->fd, -(sizeof(sdbEcommit)), SEEK_END); + while (offset > 0) { + read(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); + if (sdbEcommit == SDB_ENDCOMMIT) { + ftruncate(pTable->fd, offset + sizeof(sdbEcommit)); + break; + } + offset = lseek(pTable->fd, -(sizeof(sdbEcommit) + 1), SEEK_CUR); + } + lseek(pTable->fd, 0, SEEK_SET); + + ssize_t tsize = read(pTable->fd, &(pTable->header), size); + if (tsize < size) { + sdbError("table:%s, failed to read sdb file header, file:%s", pTable->tableName, pTable->fileName); + tclose(pTable->fd); + return -1; + } + + if (pTable->header.swVersion != swVersion.iversion) { + sdbWarn("table:%s, sdb file:%s version not match software version", pTable->tableName, pTable->fileName); + } + + if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) { + sdbError("table:%s, sdb file header is broken since checksum mismatch, file:%s", pTable->tableName, pTable->fileName); + tclose(pTable->fd); + return -1; + } + + pTable->fileSize += size; + // skip end commit symbol + lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR); + pTable->fileSize += sizeof(sdbEcommit); + } + + pTable->numOfRows = 0; + + return pTable->fd; +} + +static int32_t sdbInitTableByFile(SSdbTable *pTable) { + sdbTrace("table:%s, open sdb file:%s for read", pTable->tableName, pTable->fileName); + if (sdbOpenSdbFile(pTable) < 0) { + sdbError("table:%s, failed to open sdb file:%s for read", pTable->tableName, pTable->fileName); + return -1; + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)malloc(total_size); + if (rowHead == NULL) { + sdbError("table:%s, failed to allocate row head memory, sdb:%s", pTable->tableName, pTable->tableName); + return -1; + } + + int32_t numOfChanged = 0; + int32_t maxAutoIndex = 0; + while (1) { + memset(rowHead, 0, total_size); + + int32_t bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); + if (bytes < 0) { + sdbError("table:%s, failed to read sdb file:%s", pTable->tableName, pTable->fileName); + tfree(rowHead); + return -1; + } + + if (bytes == 0) break; + + if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { + pTable->fileSize++; + lseek(pTable->fd, -(bytes - 1), SEEK_CUR); + continue; + } + + if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { + sdbError("table:%s, error row size in sdb filesize:%d, version:%d rowSize:%d maxRowSize:%d", pTable->tableName, + pTable->fileSize, rowHead->version, rowHead->rowSize, pTable->maxRowSize); + pTable->fileSize += sizeof(SRowHead); + continue; + } + + bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); + if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { + // TODO: Here may cause pTable->fileSize not end of the file + sdbError("table:%s, failed to read sdb file, version:%d rowSize:%d", pTable->tableName, rowHead->version, + rowHead->rowSize); + break; + } + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { + sdbError("table:%s, error sdb checksum, version:%d, skip", pTable->tableName, rowHead->version); + pTable->fileSize += real_size; + continue; + } + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); + } + + pTable->version = MAX(pTable->version, abs(rowHead->version)); + + void *pMetaRow = sdbGetRow(pTable, rowHead->data); + if (pMetaRow == NULL) { + if (rowHead->version < 0) { + sdbError("table:%s, error sdb negative version:%d, record:%s, skip", pTable->tableName, rowHead->version, + sdbGetkeyStr(pTable, rowHead->data)); + } else { + SRowMeta rowMeta; + rowMeta.version = rowHead->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = rowHead->rowSize; + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .rowSize = rowHead->rowSize + }; + int32_t code = (*pTable->decodeFp)(&oper); + if (code == TSDB_CODE_SUCCESS) { + rowMeta.row = oper.pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); + pTable->numOfRows++; + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read new record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } else { + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } + } + } else { + if (rowHead->version < 0) { + SSdbOperDesc oper = { + .table = pTable, + .pObj = pMetaRow + }; + (*pTable->destroyFp)(&oper); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); + pTable->numOfRows--; + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read deleted record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } else { + SRowMeta rowMeta; + rowMeta.version = rowHead->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = rowHead->rowSize; + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .rowSize = rowHead->rowSize, + .pObj = pMetaRow + }; + + (*pTable->destroyFp)(&oper); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); + + int32_t code = (*pTable->decodeFp)(&oper); + if (code == TSDB_CODE_SUCCESS) { + rowMeta.row = oper.pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read updated record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } else { + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } + } + numOfChanged++; + } + + pTable->fileSize += real_size; + pTable->fileSize += 4; + lseek(pTable->fd, 4, SEEK_CUR); + } + + void *pNode = NULL; + while (1) { + SRowMeta * pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; + + SSdbOperDesc oper = { + .pObj = pMeta->row, + .table = pTable, + .version = pMeta->version, + }; + + int32_t code = (*pTable->insertFp)(&oper); + if (code != TSDB_CODE_SUCCESS) { + sdbError("table:%s, failed to insert record:%s", pTable->tableName, sdbGetkeyStr(pTable, rowHead->data)); + } + } + + sdbVersion += pTable->version; + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + pTable->autoIndex = maxAutoIndex; + } + + tfree(rowHead); + return 0; +} + +void *sdbOpenTable(SSdbTableDesc *pDesc) { + SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); + if (pTable == NULL) return NULL; + + pTable->keyType = pDesc->keyType; + pTable->hashSessions = pDesc->hashSessions; + pTable->maxRowSize = pDesc->maxRowSize; + pTable->insertFp = pDesc->insertFp; + pTable->deleteFp = pDesc->deleteFp; + pTable->updateFp = pDesc->updateFp; + pTable->encodeFp = pDesc->encodeFp; + pTable->decodeFp = pDesc->decodeFp; + pTable->destroyFp = pDesc->destroyFp; + strcpy(pTable->tableName, pDesc->tableName); + sprintf(pTable->fileName, "%s/%s.db", tsMnodeDir, pTable->tableName); + + if (sdbInitIndexFp[pTable->keyType] != NULL) { + pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); + } + + pthread_mutex_init(&pTable->mutex, NULL); + + if (sdbInitTableByFile(pTable) < 0) return NULL; + + pTable->tableId = sdbNumOfTables++; + sdbTableList[pTable->tableId] = pTable; + + sdbTrace("table:%s, is initialized, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d numOfTables:%d", + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbNumOfTables); + + return pTable; +} + +static SRowMeta *sdbGetRowMeta(void *handle, void *key) { + SSdbTable *pTable = (SSdbTable *)handle; + SRowMeta * pMeta; + + if (handle == NULL) return NULL; + + pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); + + return pMeta; +} + +void *sdbGetRow(void *handle, void *key) { + SSdbTable *pTable = (SSdbTable *)handle; + SRowMeta * pMeta; + + if (handle == NULL) return NULL; + + pthread_mutex_lock(&pTable->mutex); + pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); + pthread_mutex_unlock(&pTable->mutex); + + if (pMeta == NULL) { + return NULL; + } + + return pMeta->row; +} + +int32_t sdbInsertRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) { + sdbError("sdb tables is null"); + return TSDB_CODE_OTHERS; + } + + if (sdbGetRow(pTable, pOper->pObj)) { + sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj)); + return TSDB_CODE_ALREADY_THERE; + } + + pOper->maxRowSize = pTable->maxRowSize; + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_INSERT, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to forward record:%s version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); + pthread_mutex_unlock(&pTable->mutex); + return TSDB_CODE_OTHERS; + } + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + pthread_mutex_unlock(&pTable->mutex); + sdbError("table:%s, failed to allocate row head memory for record:%s version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); + return -1; + } + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; + } + pTable->version++; + sdbVersion++; + + pOper->rowData = rowHead->data; + (*pTable->encodeFp)(pOper); + rowHead->rowSize = pOper->rowSize; + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; + assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("table:%s, failed to get checksum while inserting", pTable->tableName); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + tfree(rowHead); + + // update in SDB layer + SRowMeta rowMeta; + rowMeta.version = pTable->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = pOper->rowSize; + rowMeta.row = pOper->pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); + + pTable->numOfRows++; + + pthread_mutex_unlock(&pTable->mutex); + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, insert record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + + (*pTable->insertFp)(pOper); + + return 0; +} + +// row here can be object or null-terminated string +int32_t sdbDeleteRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) return -1; + + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); + if (pMeta == NULL) { + sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); + return -1; + } + + void * pMetaRow = pMeta->row; + assert(pMetaRow != NULL); + + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_DELETE, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pMeta->rowSize, + .rowData = pMeta->row, + }; + + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to delete record", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + } + + int32_t total_size = sizeof(SRowHead) + pMeta->rowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + + pTable->version++; + sdbVersion++; + + int32_t rowSize = 0; + switch (pTable->keyType) { + case SDB_KEY_TYPE_STRING: + rowSize = strlen((char *)pOper->pObj) + 1; + break; + case SDB_KEY_TYPE_AUTO: + rowSize = sizeof(uint64_t); + break; + default: + return -1; + } + + rowHead->delimiter = SDB_DELIMITER; + rowHead->rowSize = rowSize; + rowHead->version = -(pTable->version); + memcpy(rowHead->data, pOper->pObj, rowSize); + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + + tfree(rowHead); + + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, delete record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + + // Delete from current layer + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); + + pTable->numOfRows--; + + pthread_mutex_unlock(&pTable->mutex); + + (*pTable->deleteFp)(pOper); + (*pTable->destroyFp)(pOper); + + return 0; +} + +// row here can be the object or the string info (encoded string) +int32_t sdbUpdateRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) return -1; + + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); + if (pMeta == NULL) { + sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " version:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version); + return -1; + } + + void *pMetaRow = pMeta->row; + assert(pMetaRow != NULL); + + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_UPDATE, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to update record", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + sdbError("table:%s, failed to allocate row head memory", pTable->tableName); + return -1; + } + + if (pMetaRow != pOper->pObj) { + memcpy(rowHead->data, pOper->rowData, pOper->rowSize); + rowHead->rowSize = pOper->rowSize; + } else { + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .maxRowSize = pTable->maxRowSize, + .pObj = pOper->pObj + }; + (*pTable->encodeFp)(&oper); + rowHead->rowSize = oper.rowSize; + } + + pTable->version++; + sdbVersion++; + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("table:%s, failed to get checksum, version:%d", pTable->tableName, rowHead->version); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, update record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + + pMeta->version = pTable->version; + pMeta->offset = pTable->fileSize; + pMeta->rowSize = rowHead->rowSize; + + pthread_mutex_unlock(&pTable->mutex); + + (*pTable->updateFp)(pOper); // update in upper layer + + tfree(rowHead); + + return 0; +} + +void sdbCloseTable(void *handle) { + SSdbTable *pTable = (SSdbTable *)handle; + void * pNode = NULL; + + if (pTable == NULL) return; + + while (1) { + SRowMeta * pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; + + SSdbOperDesc oper = { + .pObj = pMeta->row, + .table = pTable, + .version = pMeta->version, + }; + + (*pTable->destroyFp)(&oper); + } + + if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); + + if (pTable->fd) tclose(pTable->fd); + + pthread_mutex_destroy(&pTable->mutex); + + sdbNumOfTables--; + sdbTrace("table:%s, is closed, version:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); + + tfree(pTable); +} + +void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { + SSdbTable *pTable = (SSdbTable *)handle; + SRowMeta * pMeta; + + *ppRow = NULL; + if (pTable == NULL) return NULL; + + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) return NULL; + + *ppRow = pMeta->row; + + return pNode; +} diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 9cf0e3b57338f113996579d9cb129013691fdea9..b092445e7278936634658bb2220637e664ad198b 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -30,8 +30,8 @@ #include "mgmtDnode.h" #include "mgmtGrant.h" #include "mgmtMnode.h" -#include "mgmtNormalTable.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" @@ -63,7 +63,7 @@ int32_t mgmtInitShell() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_HEARTBEAT, mgmtProcessHeartBeatMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg); - tsMgmtTranQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, 1, "mnodeT"); + tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT"); int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0; if (numOfThreads < 1) { @@ -131,19 +131,25 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) { } static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { - if (sdbGetRunStatus() != SDB_STATUS_SERVING) { + if (!mgmtInServerStatus()) { mgmtProcessMsgWhileNotReady(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; } + if (mgmtCheckExpired()) { + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_GRANT_EXPIRED); + return; + } + if (tsMgmtProcessShellMsgFp[rpcMsg->msgType] == NULL) { mgmtProcessUnSupportMsg(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; } - SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle); + bool usePublicIp = false; + SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle, &usePublicIp); if (pUser == NULL) { mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_INVALID_USER); rpcFreeCont(rpcMsg->pCont); @@ -157,6 +163,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { queuedMsg.contLen = rpcMsg->contLen; queuedMsg.pCont = rpcMsg->pCont; queuedMsg.pUser = pUser; + queuedMsg.usePublicIp = usePublicIp; (*tsMgmtProcessShellMsgFp[rpcMsg->msgType])(&queuedMsg); rpcFreeCont(rpcMsg->pCont); } else { @@ -166,6 +173,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { queuedMsg->contLen = rpcMsg->contLen; queuedMsg->pCont = rpcMsg->pCont; queuedMsg->pUser = pUser; + queuedMsg->usePublicIp = usePublicIp; mgmtAddToShellQueue(queuedMsg); } } @@ -309,20 +317,10 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { return; } - pHBRsp->ipList.inUse = 0; - pHBRsp->ipList.port = htons(tsMnodeShellPort); - pHBRsp->ipList.numOfIps = 0; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - pHBRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps); - if (connInfo.serverIp == tsPublicIpInt) { - for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) { - pHBRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]); - } - } else { - for (int i = 0; i < pSdbIpList->numOfIps; ++i) { - pHBRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]); - } - } + if (connInfo.serverIp == tsPublicIpInt) { + mgmtGetMnodePublicIpList(&pHBRsp->ipList); + } else { + mgmtGetMnodePrivateIpList(&pHBRsp->ipList); } /* @@ -411,20 +409,11 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { strcpy(pConnectRsp->serverVersion, version); pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->superAuth = pUser->superAuth; - pConnectRsp->ipList.inUse = 0; - pConnectRsp->ipList.port = htons(tsMnodeShellPort); - pConnectRsp->ipList.numOfIps = 0; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - pConnectRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps); - if (connInfo.serverIp == tsPublicIpInt) { - for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) { - pConnectRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]); - } - } else { - for (int i = 0; i < pSdbIpList->numOfIps; ++i) { - pConnectRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]); - } - } + + if (connInfo.serverIp == tsPublicIpInt) { + mgmtGetMnodePublicIpList(&pConnectRsp->ipList); + } else { + mgmtGetMnodePrivateIpList(&pConnectRsp->ipList); } connect_over: @@ -458,7 +447,7 @@ static bool mgmtCheckMeterMetaMsgType(void *pMsg) { static bool mgmtCheckMsgReadOnly(int8_t type, void *pCont) { if ((type == TSDB_MSG_TYPE_CM_TABLE_META && (!mgmtCheckMeterMetaMsgType(pCont))) || - type == TSDB_MSG_TYPE_CM_STABLE_META || type == TSDB_MSG_TYPE_RETRIEVE || + type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_RETRIEVE || type == TSDB_MSG_TYPE_CM_SHOW || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_CONNECT) { return true; diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index bd697ade93288917e38bb1388049d98717bdd9c3..306d414cbcfaf20cc37d0e94428ca010cc6b38e3 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -15,178 +15,126 @@ #define _DEFAULT_SOURCE #include "os.h" - +#include "name.h" +#include "tsqlfunction.h" #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDb.h" +#include "mgmtDClient.h" #include "mgmtDnode.h" #include "mgmtGrant.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" +#include "mgmtSdb.h" #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" -#include "mnode.h" - -#include "name.h" -#include "tsqlfunction.h" -static void *tsSuperTableSdb; +static void *tsSuperTableSdb; static int32_t tsSuperTableUpdateSize; - -static void *(*mgmtSuperTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); +static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *queueMsg); +static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg); static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static void mgmtDestroySuperTable(SSuperTableObj *pTable) { - free(pTable->schema); - free(pTable); -} - -static void mgmtSuperTableActionInit() { - SSuperTableObj tObj; - tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - mgmtSuperTableActionFp[SDB_TYPE_INSERT] = mgmtSuperTableActionInsert; - mgmtSuperTableActionFp[SDB_TYPE_DELETE] = mgmtSuperTableActionDelete; - mgmtSuperTableActionFp[SDB_TYPE_UPDATE] = mgmtSuperTableActionUpdate; - mgmtSuperTableActionFp[SDB_TYPE_ENCODE] = mgmtSuperTableActionEncode; - mgmtSuperTableActionFp[SDB_TYPE_DECODE] = mgmtSuperTableActionDecode; - mgmtSuperTableActionFp[SDB_TYPE_RESET] = mgmtSuperTableActionReset; - mgmtSuperTableActionFp[SDB_TYPE_DESTROY] = mgmtSuperTableActionDestroy; -} - -void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - memcpy(pTable, str, tsSuperTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = realloc(pTable->schema, schemaSize); - memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize); - - return NULL; +static void mgmtDestroySuperTable(SSuperTableObj *pStable) { + tfree(pStable->schema); + tfree(pStable); } -void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - mgmtDestroySuperTable(pTable); - return NULL; +static int32_t mgmtSuperTableActionDestroy(SSdbOperDesc *pOper) { + mgmtDestroySuperTable(pOper->pObj); + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - STableInfo *pTable = (STableInfo *) row; - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb) { +static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) { + SSuperTableObj *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); + if (pDb != NULL) { mgmtAddSuperTableIntoDb(pDb); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - STableInfo *pTable = (STableInfo *) row; - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb) { +static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) { + SSuperTableObj *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); + if (pDb != NULL) { mgmtRemoveSuperTableFromDb(pDb); + mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtSuperTableActionReset(row, str, size, NULL); +static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) { + SSuperTableObj *pStable = pOper->pObj; + assert(pOper->pObj != NULL && pOper->rowData != NULL); - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - if (size < tsSuperTableUpdateSize + schemaSize + 1) { - *ssize = -1; - return NULL; + if (pOper->maxRowSize < tsSuperTableUpdateSize + schemaSize) { + return TSDB_CODE_INVALID_MSG_LEN; } - memcpy(str, pTable, tsSuperTableUpdateSize); - memcpy(str + tsSuperTableUpdateSize, pTable->schema, schemaSize); - *ssize = tsSuperTableUpdateSize + schemaSize; + memcpy(pOper->rowData, pStable, tsSuperTableUpdateSize); + memcpy(pOper->rowData + tsSuperTableUpdateSize, pStable->schema, schemaSize); + pOper->rowSize = tsSuperTableUpdateSize + schemaSize; - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - SSuperTableObj *pTable = (SSuperTableObj *) malloc(sizeof(SSuperTableObj)); - if (pTable == NULL) { - return NULL; - } - memset(pTable, 0, sizeof(SSuperTableObj)); + SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj)); + if (pStable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - if (size < tsSuperTableUpdateSize) { - mgmtDestroySuperTable(pTable); - return NULL; - } - memcpy(pTable, str, tsSuperTableUpdateSize); + memcpy(pStable, pOper->rowData, tsSuperTableUpdateSize); - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = malloc(schemaSize); - if (pTable->schema == NULL) { - mgmtDestroySuperTable(pTable); - return NULL; + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); + pStable->schema = malloc(schemaSize); + if (pStable->schema == NULL) { + mgmtDestroySuperTable(pStable); + return -1; } - memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize); - return (void *) pTable; -} + memcpy(pStable->schema, pOper->rowData + tsSuperTableUpdateSize, schemaSize); + pOper->pObj = pStable; -void *mgmtSuperTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtSuperTableActionFp[(uint8_t) action] != NULL) { - return (*(mgmtSuperTableActionFp[(uint8_t) action]))(row, str, size, ssize); - } - return NULL; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitSuperTables() { - void *pNode = NULL; - void *pLastNode = NULL; - SSuperTableObj *pTable = NULL; - - mgmtSuperTableActionInit(); - - tsSuperTableSdb = sdbOpenTable(tsMaxTables, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, - "stables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtSuperTableAction); + SSuperTableObj tObj; + tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "stables", + .hashSessions = TSDB_MAX_SUPER_TABLES, + .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtSuperTableActionInsert, + .deleteFp = mgmtSuperTableActionDelete, + .updateFp = mgmtSuperTableActionUpdate, + .encodeFp = mgmtSuperTableActionEncode, + .decodeFp = mgmtSuperTableActionDecode, + .destroyFp = mgmtSuperTableActionDestroy, + }; + + tsSuperTableSdb = sdbOpenTable(&tableDesc); if (tsSuperTableSdb == NULL) { mError("failed to init stables data"); return -1; } - pNode = NULL; - while (1) { - pNode = sdbFetchRow(tsSuperTableSdb, pNode, (void **) &pTable); - if (pTable == NULL) { - break; - } - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("super table:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsSuperTableSdb, pTable); - pNode = pLastNode; - continue; - } - - mgmtAddSuperTableIntoDb(pDb); - } - + mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_STABLE_VGROUP, mgmtProcessSuperTableVgroupMsg); mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mgmtGetShowSuperTableMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mgmtRetrieveShowSuperTables); + mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropStableRsp); mTrace("stables is initialized"); return 0; @@ -196,24 +144,18 @@ void mgmtCleanUpSuperTables() { sdbCloseTable(tsSuperTableSdb); } -int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { - int32_t numOfTables = sdbGetNumOfRows(tsSuperTableSdb); - if (numOfTables >= TSDB_MAX_SUPER_TABLES) { - mError("stable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_SUPER_TABLES); - return TSDB_CODE_TOO_MANY_TABLES; - } - - SSuperTableObj *pStable = (SSuperTableObj *)calloc(sizeof(SSuperTableObj), 1); +void mgmtCreateSuperTable(SQueuedMsg *pMsg) { + SCMCreateTableMsg *pCreate = pMsg->pCont; + SSuperTableObj *pStable = (SSuperTableObj *)calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) { - return TSDB_CODE_SERV_OUT_OF_MEMORY; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; } - strcpy(pStable->tableId, pCreate->tableId); - pStable->type = TSDB_SUPER_TABLE; + strcpy(pStable->info.tableId, pCreate->tableId); + pStable->info.type = TSDB_SUPER_TABLE; pStable->createdTime = taosGetTimestampMs(); - pStable->vgId = 0; - pStable->sid = 0; - pStable->uid = (((uint64_t) pStable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + pStable->uid = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pStable->sversion = 0; pStable->numOfColumns = htons(pCreate->numOfColumns); pStable->numOfTags = htons(pCreate->numOfTags); @@ -224,7 +166,8 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { if (pStable->schema == NULL) { free(pStable); mError("stable:%s, no schema input", pCreate->tableId); - return TSDB_CODE_INVALID_TABLE; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + return; } memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); @@ -235,24 +178,36 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { tschema[col].bytes = htons(tschema[col].bytes); } - if (sdbInsertRow(tsSuperTableSdb, pStable, 0) < 0) { - mError("stable:%s, update sdb error", pStable->tableId); - return TSDB_CODE_SDB_ERROR; - } + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable, + .rowSize = sizeof(SSuperTableObj) + schemaSize + }; - mPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); - return TSDB_CODE_SUCCESS; + int32_t code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mgmtDestroySuperTable(pStable); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR); + } else { + mLPrint("stable:%s, is created, tags:%d cols:%d", pStable->info.tableId, pStable->numOfTags, pStable->numOfColumns); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS); + } } -int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pStable) { +void mgmtDropSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pStable) { if (pStable->numOfTables != 0) { - mError("stable:%s, numOfTables:%d not 0", pStable->tableId, pStable->numOfTables); - return TSDB_CODE_OTHERS; + mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); } else { - //TODO: drop child tables - mError("stable:%s, is dropped from sdb", pStable->tableId); - mgmtRemoveSuperTableFromDb(pDb); - return TSDB_CODE_OTHERS; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable + }; + int32_t code = sdbDeleteRow(&oper); + mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code)); + mgmtSendSimpleResp(pMsg->thandle, code); } } @@ -260,15 +215,14 @@ void* mgmtGetSuperTable(char *tableId) { return sdbGetRow(tsSuperTableSdb, tableId); } -void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) { - //TODO get vgroup of dnodes - SCMSuperTableInfoRsp *rsp = rpcMallocCont(sizeof(SCMSuperTableInfoRsp) + sizeof(uint32_t) * mgmtGetDnodesNum()); - rsp->numOfDnodes = 1; - rsp->dnodeIps[0] = 0; +static void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) { + SCMSTableVgroupRspMsg *rsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * mgmtGetDnodesNum()); + rsp->numOfDnodes = htonl(1); + rsp->dnodeIps[0] = htonl(inet_addr(tsPrivateIp)); return rsp; } -int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) { +static int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) { for (int32_t i = 0; i < pStable->numOfTags; i++) { SSchema *schema = (SSchema *)(pStable->schema + (pStable->numOfColumns + i) * sizeof(SSchema)); if (strcasecmp(tagName, schema->name) == 0) { @@ -279,7 +233,7 @@ int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) return -1; } -int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { +static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { return TSDB_CODE_APP_ERROR; } @@ -293,9 +247,9 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t } } - SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); if (pDb == NULL) { - mError("meter: %s not belongs to any database", pStable->tableId); + mError("meter: %s not belongs to any database", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } @@ -321,21 +275,21 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ntags * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); - mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->tableId); + mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->info.tableId); return TSDB_CODE_SUCCESS; } -int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { +static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t col = mgmtFindSuperTableTagIndex(pStable, tagName); if (col <= 0 || col >= pStable->numOfTags) { return TSDB_CODE_APP_ERROR; } - SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); if (pDb == NULL) { - mError("table: %s not belongs to any database", pStable->tableId); + mError("table: %s not belongs to any database", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } @@ -354,21 +308,21 @@ int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } -int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) { +static int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) { int32_t col = mgmtFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { // Tag name does not exist - mError("Failed to modify table %s tag column, oname: %s, nname: %s", pStable->tableId, oldTagName, newTagName); + mError("Failed to modify table %s tag column, oname: %s, nname: %s", pStable->info.tableId, oldTagName, newTagName); return TSDB_CODE_INVALID_MSG_TYPE; } - int32_t rowSize = 0; - uint32_t len = strlen(newTagName); + // int32_t rowSize = 0; + uint32_t len = strlen(newTagName); if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { return TSDB_CODE_APP_ERROR; @@ -384,17 +338,18 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN if (msg == NULL) return TSDB_CODE_APP_ERROR; memset(msg, 0, size); - mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); + // mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); - int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, rowSize, 1); + int32_t ret = 0; + // int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); tfree(msg); if (ret < 0) { - mError("Failed to modify table %s tag column", pStable->tableId); + mError("Failed to modify table %s tag column", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } - mTrace("Succeed to modify table %s tag column", pStable->tableId); + mTrace("Succeed to modify table %s tag column", pStable->info.tableId); return TSDB_CODE_SUCCESS; } @@ -409,7 +364,7 @@ static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colN return -1; } -int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { +static int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { if (ncols <= 0) { return TSDB_CODE_APP_ERROR; } @@ -420,9 +375,9 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32 } } - SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); if (pDb == NULL) { - mError("meter: %s not belongs to any database", pStable->tableId); + mError("meter: %s not belongs to any database", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } @@ -448,20 +403,20 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32 pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } -int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { +static int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { int32_t col = mgmtFindSuperTableColumnIndex(pStable, colName); if (col < 0) { return TSDB_CODE_APP_ERROR; } - SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); if (pDb == NULL) { - mError("table: %s not belongs to any database", pStable->tableId); + mError("table: %s not belongs to any database", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } @@ -481,7 +436,7 @@ int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { pStable->schema = realloc(pStable->schema, schemaSize); pAcct->acctInfo.numOfTimeSeries -= (pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -548,7 +503,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v SDbObj *pDb = mgmtGetDb(pShow->db); if (pDb == NULL) return 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { if (strcmp(pUser->user, "root") != 0 && strcmp(pUser->user, "_root") != 0 && strcmp(pUser->user, "monitor") != 0 ) { @@ -566,12 +521,12 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v while (numOfRows < rows) { pShow->pNode = sdbFetchRow(tsSuperTableSdb, pShow->pNode, (void **) &pTable); if (pTable == NULL) break; - if (strncmp(pTable->tableId, prefix, prefixLen)) { + if (strncmp(pTable->info.tableId, prefix, prefixLen)) { continue; } memset(stableName, 0, tListLen(stableName)); - extractTableName(pTable->tableId, stableName); + mgmtExtractTableName(pTable->info.tableId, stableName); if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) @@ -619,23 +574,20 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { break; } - if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsSuperTableSdb, pTable); + if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsSuperTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; numOfTables ++; continue; } } - mTrace("db:%s, all super tables:%d is dropped", pDropDb->name, numOfTables); -} - -void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable) { - pStable->numOfTables++; -} - -void mgmtRemoveTableFromSuperTable(SSuperTableObj *pStable) { - pStable->numOfTables--; + mTrace("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { @@ -651,18 +603,70 @@ int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { return (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema); } -int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { +void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable) { + SDbObj *pDb = pMsg->pDb; + + STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); pMeta->uid = htobe64(pTable->uid); - pMeta->sid = htonl(pTable->sid); - pMeta->vgId = htonl(pTable->vgId); pMeta->sversion = htons(pTable->sversion); pMeta->precision = pDb->cfg.precision; - pMeta->numOfTags = pTable->numOfTags; - pMeta->numOfColumns = htons(pTable->numOfColumns); - pMeta->tableType = pTable->type; + pMeta->numOfTags = (uint8_t)pTable->numOfTags; + pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); + pMeta->tableType = pTable->info.type; pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable); - strcpy(pMeta->tableId, pTable->tableId); + strcpy(pMeta->tableId, pTable->info.tableId); - return TSDB_CODE_SUCCESS; + SRpcMsg rpcRsp = { + .handle = pMsg->thandle, + .pCont = pMeta, + .contLen = pMeta->contLen, + }; + pMeta->contLen = htons(pMeta->contLen); + rpcSendResponse(&rpcRsp); + + mTrace("stable:%%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid); +} + +static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { + SCMSTableVgroupMsg *pInfo = pMsg->pCont; + STableInfo *pTable = mgmtGetSuperTable(pInfo->tableId); + if (pTable == NULL) { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + return; + } + + SCMSTableVgroupRspMsg *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable); + if (pRsp != NULL) { + int32_t msgLen = sizeof(SSuperTableObj) + htonl(pRsp->numOfDnodes) * sizeof(int32_t); + SRpcMsg rpcRsp = {0}; + rpcRsp.handle = pMsg->thandle; + rpcRsp.pCont = pRsp; + rpcRsp.contLen = msgLen; + rpcSendResponse(&rpcRsp); + } else { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + } +} + +void mgmtAlterSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pTable) { + int32_t code = TSDB_CODE_OPS_NOT_SUPPORT; + SCMAlterTableMsg *pAlter = pMsg->pCont; + + if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { + code = mgmtAddSuperTableTag((SSuperTableObj *) pTable, pAlter->schema, 1); + } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { + code = mgmtDropSuperTableTag((SSuperTableObj *) pTable, pAlter->schema[0].name); + } else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { + code = mgmtModifySuperTableTagNameByName((SSuperTableObj *) pTable, pAlter->schema[0].name, pAlter->schema[1].name); + } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { + code = mgmtAddSuperTableColumn((SSuperTableObj *) pTable, pAlter->schema, 1); + } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { + code = mgmtDropSuperTableColumnByName((SSuperTableObj *) pTable, pAlter->schema[0].name); + } else {} + + mgmtSendSimpleResp(pMsg->thandle, code); } +static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) { + mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); +} \ No newline at end of file diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 2fe6e31306ecb49e81f31b6fab8c3d93843ccab5..5169f4a8174ae6d88d9a4741fcdfa8f8ac01d629 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -15,51 +15,25 @@ #define _DEFAULT_SOURCE #include "os.h" - -#include "mgmtTable.h" #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDClient.h" #include "mgmtDb.h" #include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtGrant.h" #include "mgmtMnode.h" -#include "mgmtNormalTable.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" +#include "mgmtTable.h" #include "mgmtUser.h" -#include "mgmtVgroup.h" -#include "mnode.h" - -#include "qast.h" -#include "qextbuffer.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "tscompression.h" -#include "tskiplist.h" -#include "tsqlfunction.h" -#include "tstatus.h" -#include "ttime.h" -#include "name.h" - -extern void *tsNormalTableSdb; -extern void *tsChildTableSdb; static void mgmtProcessCreateTableMsg(SQueuedMsg *queueMsg); static void mgmtProcessDropTableMsg(SQueuedMsg *queueMsg); static void mgmtProcessAlterTableMsg(SQueuedMsg *queueMsg); static void mgmtProcessTableMetaMsg(SQueuedMsg *queueMsg); -static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *queueMsg); -static void mgmtProcessSuperTableMetaMsg(SQueuedMsg *queueMsg); -static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg); -static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg); -static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg); -static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg); -static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle); int32_t mgmtInitTables() { int32_t code = mgmtInitSuperTables(); @@ -67,46 +41,26 @@ int32_t mgmtInitTables() { return code; } - code = mgmtInitNormalTables(); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - code = mgmtInitChildTables(); if (code != TSDB_CODE_SUCCESS) { return code; } - mgmtSetVgroupIdPool(); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_TABLE, mgmtProcessCreateTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_TABLE, mgmtProcessDropTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_TABLE, mgmtProcessAlterTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_TABLE_META, mgmtProcessTableMetaMsg); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_TABLES_META, mgmtProcessMultiTableMetaMsg); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_STABLE_META, mgmtProcessSuperTableMetaMsg); - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mgmtGetShowTableMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mgmtRetrieveShowTables); - mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP, mgmtProcessCreateTableRsp); - mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropTableRsp); - mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp); - mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropStableRsp); return TSDB_CODE_SUCCESS; } -STableInfo* mgmtGetTable(char *tableId) { - STableInfo *tableInfo = (STableInfo *) mgmtGetSuperTable(tableId); - if (tableInfo != NULL) { - return tableInfo; - } - - tableInfo = (STableInfo *) mgmtGetNormalTable(tableId); +STableInfo *mgmtGetTable(char *tableId) { + STableInfo *tableInfo = mgmtGetSuperTable(tableId); if (tableInfo != NULL) { return tableInfo; } - tableInfo = (STableInfo *) mgmtGetChildTable(tableId); + tableInfo = mgmtGetChildTable(tableId); if (tableInfo != NULL) { return tableInfo; } @@ -114,252 +68,29 @@ STableInfo* mgmtGetTable(char *tableId) { return NULL; } -STableInfo* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) { - SDnodeObj *pObj = mgmtGetDnode(dnodeId); - SVgObj *pVgroup = mgmtGetVgroup(vnode); - - if (pObj == NULL || pVgroup == NULL) { - return NULL; - } - - return pVgroup->tableList[sid]; -} - -int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMetaMsg *pMeta, bool usePublicIp) { - if (pTable->type == TSDB_CHILD_TABLE) { - mgmtGetChildTableMeta(pDb, (SChildTableObj *) pTable, pMeta, usePublicIp); - } else if (pTable->type == TSDB_NORMAL_TABLE) { - mgmtGetNormalTableMeta(pDb, (SNormalTableObj *) pTable, pMeta, usePublicIp); - } else if (pTable->type == TSDB_SUPER_TABLE) { - mgmtGetSuperTableMeta(pDb, (SSuperTableObj *) pTable, pMeta, usePublicIp); - } else { - mTrace("%s, uid:%" PRIu64 " table meta retrieve failed, invalid type", pTable->tableId, pTable->uid); - return TSDB_CODE_INVALID_TABLE; - } - - mTrace("%s, uid:%" PRIu64 " table meta is retrieved", pTable->tableId, pTable->uid); - return TSDB_CODE_SUCCESS; -} - -int32_t mgmtAlterTable(SDbObj *pDb, SCMAlterTableMsg *pAlter) { - STableInfo *pTable = mgmtGetTable(pAlter->tableId); - if (pTable == NULL) { - return TSDB_CODE_INVALID_TABLE; - } - - if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { - return TSDB_CODE_MONITOR_DB_FORBIDDEN; - } - - if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { - if (pTable->type == TSDB_SUPER_TABLE) { - return mgmtAddSuperTableTag((SSuperTableObj *) pTable, pAlter->schema, 1); - } - } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { - if (pTable->type == TSDB_SUPER_TABLE) { - return mgmtDropSuperTableTag((SSuperTableObj *) pTable, pAlter->schema[0].name); - } - } else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { - if (pTable->type == TSDB_SUPER_TABLE) { - return mgmtModifySuperTableTagNameByName((SSuperTableObj *) pTable, pAlter->schema[0].name, pAlter->schema[1].name); - } - } else if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { - if (pTable->type == TSDB_CHILD_TABLE) { - return mgmtModifyChildTableTagValueByName((SChildTableObj *) pTable, pAlter->schema[0].name, pAlter->tagVal); - } - } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { - if (pTable->type == TSDB_NORMAL_TABLE) { - return mgmtAddNormalTableColumn((SNormalTableObj *) pTable, pAlter->schema, 1); - } else if (pTable->type == TSDB_SUPER_TABLE) { - return mgmtAddSuperTableColumn((SSuperTableObj *) pTable, pAlter->schema, 1); - } else {} - } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { - if (pTable->type == TSDB_NORMAL_TABLE) { - return mgmtDropNormalTableColumnByName((SNormalTableObj *) pTable, pAlter->schema[0].name); - } else if (pTable->type == TSDB_SUPER_TABLE) { - return mgmtDropSuperTableColumnByName((SSuperTableObj *) pTable, pAlter->schema[0].name); - } else {} - } else {} - - return TSDB_CODE_OPS_NOT_SUPPORT; -} - void mgmtCleanUpTables() { - mgmtCleanUpNormalTables(); mgmtCleanUpChildTables(); mgmtCleanUpSuperTables(); } -int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { - SDbObj *pDb = mgmtGetDb(pShow->db); - if (pDb == NULL) { - return TSDB_CODE_DB_NOT_SELECTED; - } - - int32_t cols = 0; - SSchema *pSchema = pMeta->schema; - - pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "table_name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 8; - pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "columns"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "stable_name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pMeta->numOfColumns = htons(cols); - pShow->numOfColumns = cols; - - pShow->offset[0] = 0; - for (int32_t i = 1; i < cols; ++i) { - pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; +void mgmtExtractTableName(char* tableId, char* name) { + int pos = -1; + int num = 0; + for (pos = 0; tableId[pos] != 0; ++pos) { + if (tableId[pos] == '.') num++; + if (num == 2) break; } - pShow->numOfRows = pDb->numOfTables; - pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - - return 0; -} - -/* - * remove the hole in result set - */ -static void mgmtVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { - if (rows < capacity) { - for (int32_t i = 0; i < numOfCols; ++i) { - memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows); - } + if (num == 2) { + strcpy(name, tableId + pos + 1); } } -int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) { - SDbObj *pDb = mgmtGetDb(pShow->db); - if (pDb == NULL) return 0; - - SUserObj *pUser = mgmtGetUserFromConn(pConn); - if (pUser == NULL) return 0; - - if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { - if (strcmp(pUser->user, "root") != 0 && strcmp(pUser->user, "_root") != 0 && - strcmp(pUser->user, "monitor") != 0) { - return 0; - } - } - - int32_t numOfRows = 0; - int32_t numOfRead = 0; - int32_t cols = 0; - void *pTable = NULL; - char *pWrite = NULL; - char prefix[20] = {0}; - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - - strcpy(prefix, pDb->name); - strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); - - while (numOfRows < rows) { - int16_t numOfColumns = 0; - int64_t createdTime = 0; - char *tableId = NULL; - char *superTableId = NULL; - void *pNormalTableNode = sdbFetchRow(tsNormalTableSdb, pShow->pNode, (void **) &pTable); - if (pTable != NULL) { - SNormalTableObj *pNormalTable = (SNormalTableObj *) pTable; - pShow->pNode = pNormalTableNode; - tableId = pNormalTable->tableId; - superTableId = NULL; - createdTime = pNormalTable->createdTime; - numOfColumns = pNormalTable->numOfColumns; - } else { - void *pChildTableNode = sdbFetchRow(tsChildTableSdb, pShow->pNode, (void **) &pTable); - if (pTable != NULL) { - SChildTableObj *pChildTable = (SChildTableObj *) pTable; - pShow->pNode = pChildTableNode; - tableId = pChildTable->tableId; - superTableId = pChildTable->superTableId; - createdTime = pChildTable->createdTime; - numOfColumns = pChildTable->superTable->numOfColumns; - } else { - break; - } - } - - // not belong to current db - if (strncmp(tableId, prefix, prefixLen)) { - continue; - } - - char tableName[TSDB_TABLE_NAME_LEN] = {0}; - memset(tableName, 0, tListLen(tableName)); - numOfRead++; - - // pattern compare for meter name - extractTableName(tableId, tableName); - - if (pShow->payloadLen > 0 && - patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) { - continue; - } - - cols = 0; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strncpy(pWrite, tableName, TSDB_TABLE_NAME_LEN); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *) pWrite = createdTime; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *) pWrite = numOfColumns; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (superTableId != NULL) { - extractTableName(superTableId, pWrite); - } - cols++; - - numOfRows++; - } - - pShow->numOfReads += numOfRead; - const int32_t NUM_OF_COLUMNS = 4; - - mgmtVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow); - - return numOfRows; -} - -void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; - +static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->pCont; mTrace("table:%s, create msg is received from thandle:%p", pCreate->tableId, pMsg->thandle); - if (mgmtCheckExpired()) { - mError("table:%s, failed to create, grant expired", pCreate->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); - return; - } + if (mgmtCheckRedirect(pMsg->thandle)) return; if (!pMsg->pUser->writeAuth) { mError("table:%s, failed to create, no rights", pCreate->tableId); @@ -367,16 +98,8 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { return; } - SAcctObj *pAcct = pMsg->pUser->pAcct; - int32_t code = mgmtCheckTableLimit(pAcct, htons(pCreate->numOfColumns)); - if (code != TSDB_CODE_SUCCESS) { - mError("table:%s, failed to create, exceed the limit", pCreate->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); - return; - } - pMsg->pDb = mgmtGetDb(pCreate->db); - if (pMsg->pDb == NULL) { + if (pMsg->pDb == NULL || pMsg->pDb->dirty) { mError("table:%s, failed to create, db not selected", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; @@ -396,88 +119,19 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { } if (pCreate->numOfTags != 0) { - mTrace("table:%s, is a super table", pCreate->tableId); - code = mgmtCreateSuperTable(pCreate); - mgmtSendSimpleResp(pMsg->thandle, code); - return; - } - - code = mgmtCheckTimeSeries(pCreate->numOfColumns); - if (code != TSDB_CODE_SUCCESS) { - mError("table:%s, failed to create, timeseries exceed the limit", pCreate->tableId); - mgmtSendSimpleResp(pMsg->thandle, code); - return; - } - - SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); - memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); - pMsg->pCont = NULL; - - SVgObj *pVgroup = mgmtGetAvailableVgroup(pMsg->pDb); - if (pVgroup == NULL) { - mTrace("table:%s, start to create a new vgroup", pCreate->tableId); - mgmtCreateVgroup(newMsg); - return; - } - - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid < 0) { - mTrace("tables:%s, no enough sid in vgroup:%d", pVgroup->vgId); - mgmtCreateVgroup(newMsg); - return; - } - - SMDCreateTableMsg *pMDCreate = NULL; - if (pCreate->numOfColumns == 0) { - mTrace("table:%s, is a child table, vgroup:%d sid:%d ahandle:%p", pCreate->tableId, pVgroup->vgId, sid, pMsg); - pTable = mgmtCreateChildTable(pCreate, pVgroup, sid); - if (pTable == NULL) { - mgmtSendSimpleResp(pMsg->thandle, terrno); - return; - } - pMDCreate = mgmtBuildCreateChildTableMsg(pCreate, (SChildTableObj *) pTable); - if (pMDCreate == NULL) { - mgmtSendSimpleResp(pMsg->thandle, terrno); - return; - } + mTrace("table:%s, is a stable", pCreate->tableId); + mgmtCreateSuperTable(pMsg); } else { - mTrace("table:%s, is a normal table, vgroup:%d sid:%d ahandle:%p", pCreate->tableId, pVgroup->vgId, sid, pMsg); - pTable = mgmtCreateNormalTable(pCreate, pVgroup, sid); - if (pTable == NULL) { - mgmtSendSimpleResp(pMsg->thandle, terrno); - return; - } - pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); - if (pMDCreate == NULL) { - mgmtSendSimpleResp(pMsg->thandle, terrno); - return; - } + mTrace("table:%s, is a ctable", pCreate->tableId); + mgmtCreateChildTable(pMsg); } - - SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); - SRpcMsg rpcMsg = { - .handle = newMsg, - .pCont = pMDCreate, - .contLen = htonl(pMDCreate->contLen), - .code = 0, - .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE - }; - - newMsg->ahandle = pTable; - mgmtSendMsgToDnode(&ipSet, &rpcMsg); } -void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; - +static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { SCMDropTableMsg *pDrop = pMsg->pCont; mTrace("table:%s, drop table msg is received from thandle:%p", pDrop->tableId, pMsg->thandle); - if (mgmtCheckExpired()) { - mError("table:%s, failed to drop, grant expired", pDrop->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); - return; - } + if (mgmtCheckRedirect(pMsg->thandle)) return; if (!pMsg->pUser->writeAuth) { mError("table:%s, failed to drop, no rights", pDrop->tableId); @@ -485,13 +139,19 @@ void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { return; } - SDbObj *pDb = mgmtGetDbByTableId(pDrop->tableId); - if (pDb == NULL) { + pMsg->pDb = mgmtGetDbByTableId(pDrop->tableId); + if (pMsg->pDb == NULL || pMsg->pDb->dirty) { mError("table:%s, failed to drop table, db not selected", pDrop->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; } + if (mgmtCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { + mError("table:%s, failed to drop table, in monitor database", pDrop->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN); + return; + } + STableInfo *pTable = mgmtGetTable(pDrop->tableId); if (pTable == NULL) { if (pDrop->igNotExists) { @@ -505,371 +165,82 @@ void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { } } - if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { - mError("table:%s, failed to drop table, in monitor database", pDrop->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN); - return; - } - - SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); - memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); - pMsg->pCont = NULL; - int32_t code; - - switch (pTable->type) { - case TSDB_SUPER_TABLE: - mTrace("table:%s, start to drop super table", pDrop->tableId); - code = mgmtDropSuperTable(newMsg, pDb, (SSuperTableObj *) pTable); - break; - case TSDB_CHILD_TABLE: - mTrace("table:%s, start to drop child table", pDrop->tableId); - code = mgmtDropChildTable(newMsg, (SChildTableObj *) pTable); - break; - case TSDB_NORMAL_TABLE: - mTrace("table:%s, start to drop normal table", pDrop->tableId); - code = mgmtDropNormalTable(newMsg, (SNormalTableObj *) pTable); - break; - case TSDB_STREAM_TABLE: - mTrace("table:%s, start to drop stream table", pDrop->tableId); - code = mgmtDropNormalTable(newMsg, (SNormalTableObj *) pTable); - break; - default: - code = TSDB_CODE_INVALID_TABLE_TYPE; - mError("table:%s, invalid table type:%d", pDrop->tableId, pTable->type); - } - - if (code != TSDB_CODE_SUCCESS) { - free(newMsg); - mgmtSendSimpleResp(pMsg->thandle, code); + if (pTable->type == TSDB_SUPER_TABLE) { + mTrace("table:%s, start to drop stable", pDrop->tableId); + mgmtDropSuperTable(pMsg, (SSuperTableObj *)pTable); + } else { + mTrace("table:%s, start to drop ctable", pDrop->tableId); + mgmtDropChildTable(pMsg, (SChildTableObj *)pTable); } } -void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) { - return; - } - - SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle); - if (pUser == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER); - return; - } - +static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { SCMAlterTableMsg *pAlter = pMsg->pCont; + mTrace("table:%s, alter table msg is received from thandle:%p", pAlter->tableId, pMsg->thandle); - int32_t code; - if (!pUser->writeAuth) { - code = TSDB_CODE_NO_RIGHTS; - } else { - pAlter->type = htons(pAlter->type); - pAlter->numOfCols = htons(pAlter->numOfCols); - - if (pAlter->numOfCols > 2) { - mError("table:%s error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols); - code = TSDB_CODE_APP_ERROR; - } else { - SDbObj *pDb = mgmtGetDb(pAlter->db); - if (pDb) { - for (int32_t i = 0; i < pAlter->numOfCols; ++i) { - pAlter->schema[i].bytes = htons(pAlter->schema[i].bytes); - } - - code = mgmtAlterTable(pDb, pAlter); - if (code == 0) { - mLPrint("table:%s is altered by %s", pAlter->tableId, pUser->user); - } - } else { - code = TSDB_CODE_DB_NOT_SELECTED; - } - } - } - - mgmtSendSimpleResp(pMsg->thandle, code); -} + if (mgmtCheckRedirect(pMsg->thandle)) return; -void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle) { - SRpcMsg rpcRsp = {.handle = thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - SDbObj* pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL || pDb->dirty) { - mError("table:%s, failed to get table meta, db not selected", pTable->tableId); - rpcRsp.code = TSDB_CODE_DB_NOT_SELECTED; - rpcSendResponse(&rpcRsp); + if (!pMsg->pUser->writeAuth) { + mTrace("table:%s, failed to alter table, no rights", pAlter->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); return; } - SRpcConnInfo connInfo; - if (rpcGetConnInfo(thandle, &connInfo) != 0) { - mError("conn:%p is already released while get table meta", thandle); + pMsg->pDb = mgmtGetDbByTableId(pAlter->tableId); + if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + mError("table:%s, failed to alter table, db not selected", pAlter->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; } - bool usePublicIp = (connInfo.serverIp == tsPublicIpInt); - - STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); - rpcRsp.code = mgmtGetTableMeta(pDb, pTable, pMeta, usePublicIp); - - if (rpcRsp.code != TSDB_CODE_SUCCESS) { - rpcFreeCont(pMeta); - } else { - rpcRsp.pCont = pMeta; - rpcRsp.contLen = pMeta->contLen; - - pMeta->contLen = htons(pMeta->contLen); - } - - rpcSendResponse(&rpcRsp); -} - -void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { - SCMTableInfoMsg *pInfo = pMsg->pCont; - pInfo->createFlag = htons(pInfo->createFlag); - - SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle); - if (pUser == NULL) { - mError("table:%s, failed to get table meta, invalid user", pInfo->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER); + if (mgmtCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { + mError("table:%s, failed to alter table, its log db", pAlter->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN); return; } - STableInfo *pTable = mgmtGetTable(pInfo->tableId); + STableInfo *pTable = mgmtGetTable(pAlter->tableId); if (pTable == NULL) { - if (pInfo->createFlag != 1) { - mError("table:%s, failed to get table meta, table not exist", pInfo->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); - return; - } else { - // on demand create table from super table if table does not exists - if (mgmtCheckRedirect(pMsg->thandle)) { - mError("table:%s, failed to create table while get meta info, need redirect message", pInfo->tableId); - return; - } - - int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData); - SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen); - if (pCreateMsg == NULL) { - mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); - return; - } - - memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData)); - strcpy(pCreateMsg->tableId, pInfo->tableId); - - mError("table:%s, start to create table while get meta info", pInfo->tableId); -// mgmtCreateTable(pCreateMsg, contLen, pMsg->thandle, true); - } - } else { - mgmtProcessGetTableMeta(pTable, pMsg->thandle); - } -} - -void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { - SRpcConnInfo connInfo; - if (rpcGetConnInfo(pMsg->thandle, &connInfo) != 0) { - mError("conn:%p is already released while get mulit table meta", pMsg->thandle); + mError("table:%s, failed to alter table, table not exist", pTable->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); return; } - bool usePublicIp = (connInfo.serverIp == tsPublicIpInt); - SUserObj *pUser = mgmtGetUser(connInfo.user); - if (pUser == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER); + pAlter->numOfCols = htons(pAlter->numOfCols); + if (pAlter->numOfCols > 2) { + mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_APP_ERROR); return; } - SCMMultiTableInfoMsg *pInfo = pMsg->pCont; - pInfo->numOfTables = htonl(pInfo->numOfTables); - - int32_t totalMallocLen = 4*1024*1024; // first malloc 4 MB, subsequent reallocation as twice - SMultiTableMeta *pMultiMeta = rpcMallocCont(totalMallocLen); - if (pMultiMeta == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); - return; - } - - pMultiMeta->contLen = sizeof(SMultiTableMeta); - pMultiMeta->numOfTables = 0; - - for (int t = 0; t < pInfo->numOfTables; ++t) { - char *tableId = (char*)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN); - STableInfo *pTable = mgmtGetTable(tableId); - if (pTable == NULL) continue; - - SDbObj *pDb = mgmtGetDbByTableId(tableId); - if (pDb == NULL) continue; - - int availLen = totalMallocLen - pMultiMeta->contLen; - if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS) { - //TODO realloc - //totalMallocLen *= 2; - //pMultiMeta = rpcReMalloc(pMultiMeta, totalMallocLen); - //if (pMultiMeta == NULL) { - /// rpcSendResponse(ahandle, TSDB_CODE_SERV_OUT_OF_MEMORY, NULL, 0); - // return TSDB_CODE_SERV_OUT_OF_MEMORY; - //} else { - // t--; - // continue; - //} - } - - STableMetaMsg *pMeta = (STableMetaMsg *)(pMultiMeta->metas + pMultiMeta->contLen); - int32_t code = mgmtGetTableMeta(pDb, pTable, pMeta, usePublicIp); - if (code == TSDB_CODE_SUCCESS) { - pMultiMeta->numOfTables ++; - pMultiMeta->contLen += pMeta->contLen; - } - } - - SRpcMsg rpcRsp = {0}; - rpcRsp.handle = pMsg->thandle; - rpcRsp.pCont = pMultiMeta; - rpcRsp.contLen = pMultiMeta->contLen; - rpcSendResponse(&rpcRsp); -} - -void mgmtProcessSuperTableMetaMsg(SQueuedMsg *pMsg) { - SCMSuperTableInfoMsg *pInfo = pMsg->pCont; - STableInfo *pTable = mgmtGetSuperTable(pInfo->tableId); - if (pTable == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); - return; + for (int32_t i = 0; i < pAlter->numOfCols; ++i) { + pAlter->schema[i].bytes = htons(pAlter->schema[i].bytes); } - SCMSuperTableInfoRsp *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable); - if (pRsp != NULL) { - int32_t msgLen = sizeof(SSuperTableObj) + htonl(pRsp->numOfDnodes) * sizeof(int32_t); - SRpcMsg rpcRsp = {0}; - rpcRsp.handle = pMsg->thandle; - rpcRsp.pCont = pRsp; - rpcRsp.contLen = msgLen; - rpcSendResponse(&rpcRsp); + if (pTable->type == TSDB_SUPER_TABLE) { + mTrace("table:%s, start to alter stable", pAlter->tableId); + mgmtAlterSuperTable(pMsg, (SSuperTableObj *)pTable); } else { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + mTrace("table:%s, start to alter ctable", pAlter->tableId); + mgmtAlterChildTable(pMsg, (SChildTableObj *)pTable); } } -static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; - - SQueuedMsg *queueMsg = rpcMsg->handle; - queueMsg->received++; - - STableInfo *pTable = queueMsg->ahandle; - mTrace("table:%s, create table rsp received, thandle:%p ahandle:%p result:%s", pTable->tableId, queueMsg->thandle, - rpcMsg->handle, tstrerror(rpcMsg->code)); - - if (rpcMsg->code != TSDB_CODE_SUCCESS) { - if (pTable->type == TSDB_CHILD_TABLE) { - sdbDeleteRow(tsChildTableSdb, pTable); - } else if (pTable->type == TSDB_NORMAL_TABLE){ - sdbDeleteRow(tsNormalTableSdb, pTable); - } else {} - mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); - mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); - } else { - mTrace("table:%s, created in dnode", pTable->tableId); - if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) { - SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); - newMsg->msgType = queueMsg->msgType; - newMsg->thandle = queueMsg->thandle; - newMsg->pDb = queueMsg->pDb; - newMsg->pUser = queueMsg->pUser; - newMsg->contLen = queueMsg->contLen; - newMsg->pCont = rpcMallocCont(newMsg->contLen); - memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); - mTrace("table:%s, start to get meta", pTable->tableId); - mgmtAddToShellQueue(newMsg); - } else { - mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); - } - } - - free(queueMsg); -} - -static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg) { - mTrace("alter table rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); -} - -static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; - - SQueuedMsg *queueMsg = rpcMsg->handle; - queueMsg->received++; - - STableInfo *pTable = queueMsg->ahandle; - mTrace("table:%s, drop table rsp received, thandle:%p result:%s", pTable->tableId, queueMsg->thandle, tstrerror(rpcMsg->code)); - - if (rpcMsg->code != TSDB_CODE_SUCCESS) { - mError("table:%s, failed to drop in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); - mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); - free(queueMsg); - return; - } +static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { + SCMTableInfoMsg *pInfo = pMsg->pCont; + mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle); - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - mError("table:%s, failed to get vgroup", pTable->tableId); - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_INVALID_VGROUP_ID); - free(queueMsg); + pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId); + if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + mError("table:%s, failed to get table meta, db not selected", pInfo->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; } - if (pTable->type == TSDB_CHILD_TABLE) { - if (sdbDeleteRow(tsChildTableSdb, pTable) < 0) { - mError("table:%s, update ctables sdb error", pTable->tableId); - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - free(queueMsg); - return; - } - } else if (pTable->type == TSDB_NORMAL_TABLE){ - if (sdbDeleteRow(tsNormalTableSdb, pTable) < 0) { - mError("table:%s, update ntables sdb error", pTable->tableId); - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - free(queueMsg); - return; - } - } - - if (pVgroup->numOfTables <= 0) { - mPrint("vgroup:%d, all tables is dropped, drop vgroup", pVgroup->vgId); - mgmtDropVgroup(pVgroup, NULL); + STableInfo *pTable = mgmtGetTable(pInfo->tableId); + if (pTable == NULL || pTable->type != TSDB_SUPER_TABLE) { + mgmtGetChildTableMeta(pMsg, (SChildTableObj *)pTable); + } else { + mgmtGetSuperTableMeta(pMsg, (SSuperTableObj *)pTable); } - - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SUCCESS); - free(queueMsg); -} - -static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) { - mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); -} - -// -// -//static void mgmtProcessTableCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *thandle) { -// SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) pCont; -// pCfg->dnode = htonl(pCfg->dnode); -// pCfg->vnode = htonl(pCfg->vnode); -// pCfg->sid = htonl(pCfg->sid); -// mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// -// if (!sdbMaster) { -// mError("dnode:%s, vnode:%d, sid:%d, not master, redirect it", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0); -// return; -// } -// -// STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); -// if (pTable == NULL) { -// mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_INVALID_TABLE, NULL, 0); -// return; -// } -// -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0); -// -// //TODO -// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); -// mgmtSendCreateTableMsg(NULL, &ipSet, NULL); -//} -// \ No newline at end of file +} \ No newline at end of file diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 771c78c2229a2191cb64973a81553b026e38926f..dc2418efd94fd36be754058b1ec46cadb1505694 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -17,67 +17,105 @@ #include "os.h" #include "trpc.h" #include "ttime.h" +#include "tutil.h" #include "mgmtAcct.h" #include "mgmtGrant.h" #include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" -void *tsUserSdb = NULL; +static void *tsUserSdb = NULL; static int32_t tsUserUpdateSize = 0; -static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass); -static int32_t mgmtDropUser(SAcctObj *pAcct, char *name); -static int32_t mgmtUpdateUser(SUserObj *pUser); -static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass); +static int32_t mgmtDropUser(SAcctObj *pAcct, char *name); +static int32_t mgmtUpdateUser(SUserObj *pUser); +static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg); static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg); -static void *(*mgmtUserActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize); -static void mgmtUserActionInit(); +static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) { + tfree(pOper->pObj); + return TSDB_CODE_SUCCESS; +} -int32_t mgmtInitUsers() { - void *pNode = NULL; - SUserObj *pUser = NULL; - SAcctObj *pAcct = NULL; - int32_t numOfUsers = 0; +static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - mgmtUserActionInit(); + if (pAcct != NULL) { + mgmtAddUserIntoAcct(pAcct, pUser); + } + else { + mError("user:%s, acct:%s info not exist in sdb", pUser->user, pUser->acct); + return TSDB_CODE_INVALID_ACCT; + } - SUserObj tObj; - tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + return TSDB_CODE_SUCCESS; +} - tsUserSdb = sdbOpenTable(tsMaxUsers, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction); - if (tsUserSdb == NULL) { - mError("failed to init user data"); +static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); + + mgmtRemoveUserFromAcct(pAcct, pUser); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; + + if (pOper->maxRowSize < tsUserUpdateSize) { return -1; + } else { + memcpy(pOper->rowData, pUser, tsUserUpdateSize); + pOper->rowSize = tsUserUpdateSize; + return TSDB_CODE_SUCCESS; } +} - while (1) { - pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser); - if (pUser == NULL) break; - - pUser->prev = NULL; - pUser->next = NULL; +static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { + SUserObj *pUser = (SUserObj *) calloc(1, sizeof(SUserObj)); + if (pUser == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - pAcct = mgmtGetAcct(pUser->acct); - mgmtAddUserIntoAcct(pAcct, pUser); + memcpy(pUser, pOper->rowData, tsUserUpdateSize); + pOper->pObj = pUser; + return TSDB_CODE_SUCCESS; +} - numOfUsers++; +int32_t mgmtInitUsers() { + SUserObj tObj; + tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "users", + .hashSessions = TSDB_MAX_USERS, + .maxRowSize = tsUserUpdateSize, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtUserActionInsert, + .deleteFp = mgmtUserActionDelete, + .updateFp = mgmtUserActionUpdate, + .encodeFp = mgmtUserActionEncode, + .decodeFp = mgmtUserActionDecode, + .destroyFp = mgmtUserActionDestroy, + }; + + tsUserSdb = sdbOpenTable(&tableDesc); + if (tsUserSdb == NULL) { + mError("failed to init user data"); + return -1; } - pAcct = mgmtGetAcct("root"); + SAcctObj *pAcct = mgmtGetAcct("root"); mgmtCreateUser(pAcct, "root", "taosdata"); mgmtCreateUser(pAcct, "monitor", tsInternalPass); mgmtCreateUser(pAcct, "_root", tsInternalPass); @@ -101,16 +139,23 @@ SUserObj *mgmtGetUser(char *name) { } static int32_t mgmtUpdateUser(SUserObj *pUser) { - return sdbUpdateRow(tsUserSdb, pUser, 0, 1); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, + .rowSize = tsUserUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pUser); + code = TSDB_CODE_SDB_ERROR; + } + + return code; } static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { - int32_t numOfUsers = sdbGetNumOfRows(tsUserSdb); - if (numOfUsers >= tsMaxUsers) { - mWarn("numOfUsers:%d, exceed tsMaxUsers:%d", numOfUsers, tsMaxUsers); - return TSDB_CODE_TOO_MANY_USERS; - } - int32_t code = mgmtCheckUserLimit(pAcct); if (code != 0) { return code; @@ -131,8 +176,7 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { return code; } - pUser = malloc(sizeof(SUserObj)); - memset(pUser, 0, sizeof(SUserObj)); + pUser = calloc(1, sizeof(SUserObj)); strcpy(pUser->user, name); taosEncryptPass((uint8_t*) pass, strlen(pass), pUser->pass); strcpy(pUser->acct, pAcct->user); @@ -143,8 +187,15 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { pUser->superAuth = 1; } - code = TSDB_CODE_SUCCESS; - if (sdbInsertRow(tsUserSdb, pUser, 0) < 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj) + }; + + code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { tfree(pUser); code = TSDB_CODE_SDB_ERROR; } @@ -165,13 +216,22 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { return TSDB_CODE_NO_RIGHTS; } - sdbDeleteRow(tsUserSdb, pUser); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser + }; - return 0; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_SDB_ERROR; + } + + return code; } static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) { return TSDB_CODE_INVALID_USER; } @@ -250,86 +310,12 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void return numOfRows; } -static void mgmtUserActionInit() { - mgmtUserActionFp[SDB_TYPE_INSERT] = mgmtUserActionInsert; - mgmtUserActionFp[SDB_TYPE_DELETE] = mgmtUserActionDelete; - mgmtUserActionFp[SDB_TYPE_UPDATE] = mgmtUserActionUpdate; - mgmtUserActionFp[SDB_TYPE_ENCODE] = mgmtUserActionEncode; - mgmtUserActionFp[SDB_TYPE_DECODE] = mgmtUserActionDecode; - mgmtUserActionFp[SDB_TYPE_RESET] = mgmtUserActionReset; - mgmtUserActionFp[SDB_TYPE_DESTROY] = mgmtUserActionDestroy; -} - -static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtUserActionFp[(uint8_t) action] != NULL) { - return (*(mgmtUserActionFp[(uint8_t) action]))(row, str, size, ssize); - } - return NULL; -} - -static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - pUser->pAcct = pAcct; - mgmtAddUserIntoAcct(pAcct, pUser); - - return NULL; -} - -static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - mgmtRemoveUserFromAcct(pAcct, pUser); - - return NULL; -} - -static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtUserActionReset(row, str, size, ssize); -} - -static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - - if (size < tsUserUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pUser, tsUserUpdateSize); - *ssize = tsUserUpdateSize; - } - - return NULL; -} - -static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) malloc(sizeof(SUserObj)); - if (pUser == NULL) return NULL; - memset(pUser, 0, sizeof(SUserObj)); - - memcpy(pUser, str, tsUserUpdateSize); - - return (void *)pUser; -} - -static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *)row; - - memcpy(pUser, str, tsUserUpdateSize); - - return NULL; -} - -static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - tfree(row); - - return NULL; -} - -SUserObj *mgmtGetUserFromConn(void *pConn) { +SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) { SRpcConnInfo connInfo; if (rpcGetConnInfo(pConn, &connInfo) == 0) { + if (usePublicIp) { + *usePublicIp = (connInfo.serverIp == tsPublicIpInt); + } return mgmtGetUser(connInfo.user); } @@ -393,7 +379,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { memset(pUser->pass, 0, sizeof(pUser->pass)); taosEncryptPass((uint8_t*)pAlter->pass, strlen(pAlter->pass), pUser->pass); code = mgmtUpdateUser(pUser); - mLPrint("user:%s password is altered by %s, code:%d", pAlter->user, pUser->user, code); + mLPrint("user:%s password is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } else { code = TSDB_CODE_NO_RIGHTS; } @@ -428,10 +414,6 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } if (hasRight) { - //if (pAlter->privilege == 1) { // super - // pUser->superAuth = 1; - // pUser->writeAuth = 1; - //} if (pAlter->privilege == 2) { // read pUser->superAuth = 0; pUser->writeAuth = 0; @@ -442,7 +424,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } code = mgmtUpdateUser(pUser); - mLPrint("user:%s privilege is altered by %s, code:%d", pAlter->user, pUser->user, code); + mLPrint("user:%s privilege is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } else { code = TSDB_CODE_NO_RIGHTS; } @@ -492,7 +474,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { if (hasRight) { code = mgmtDropUser(pUser->pAcct, pDrop->user); if (code == TSDB_CODE_SUCCESS) { - mLPrint("user:%s is dropped by %s", pDrop->user, pUser->user); + mLPrint("user:%s is dropped by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } } else { code = TSDB_CODE_NO_RIGHTS; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index f84b5443a5bd289c78bf66f3672fc4626fe722d9..f6f0e9b76a31940564e3b0090bbfa23c208686e4 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -20,107 +20,156 @@ #include "tstatus.h" #include "mnode.h" #include "mgmtBalance.h" +#include "mgmtChildTable.h" #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtDnode.h" +#include "mgmtDServer.h" +#include "mgmtMnode.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtTable.h" #include "mgmtVgroup.h" -static void *tsVgroupSdb = NULL; +static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; -static void *(*mgmtVgroupActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg); static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg); +static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) ; static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); static void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); -static void mgmtVgroupActionInit() { - SVgObj tObj; - tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - mgmtVgroupActionFp[SDB_TYPE_INSERT] = mgmtVgroupActionInsert; - mgmtVgroupActionFp[SDB_TYPE_DELETE] = mgmtVgroupActionDelete; - mgmtVgroupActionFp[SDB_TYPE_UPDATE] = mgmtVgroupActionUpdate; - mgmtVgroupActionFp[SDB_TYPE_ENCODE] = mgmtVgroupActionEncode; - mgmtVgroupActionFp[SDB_TYPE_DECODE] = mgmtVgroupActionDecode; - mgmtVgroupActionFp[SDB_TYPE_RESET] = mgmtVgroupActionReset; - mgmtVgroupActionFp[SDB_TYPE_DESTROY] = mgmtVgroupActionDestroy; +static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + if (pVgroup->idPool) { + taosIdPoolCleanUp(pVgroup->idPool); + pVgroup->idPool = NULL; + } + if (pVgroup->tableList) { + tfree(pVgroup->tableList); + } + tfree(pOper->pObj); + return TSDB_CODE_SUCCESS; } -static void *mgmtVgroupAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtVgroupActionFp[(uint8_t) action] != NULL) { - return (*(mgmtVgroupActionFp[(uint8_t) action]))(row, str, size, ssize); +static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + if (pDb == NULL) { + return TSDB_CODE_INVALID_DB; } - return NULL; -} -int32_t mgmtInitVgroups() { - void *pNode = NULL; - SVgObj *pVgroup = NULL; + pVgroup->pDb = pDb; + pVgroup->prev = NULL; + pVgroup->next = NULL; - mgmtVgroupActionInit(); + int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxSessions; + pVgroup->tableList = calloc(pDb->cfg.maxSessions, sizeof(SChildTableObj *)); + if (pVgroup->tableList == NULL) { + mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size); + return -1; + } - tsVgroupSdb = sdbOpenTable(tsMaxVGroups, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction); - if (tsVgroupSdb == NULL) { - mError("failed to init vgroups data"); + pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); + if (pVgroup->idPool == NULL) { + mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId); + tfree(pVgroup->tableList); return -1; } - while (1) { - pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); - if (pVgroup == NULL) break; + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); + pVgroup->vnodeGid[i].privateIp = pDnode->privateIp; + pVgroup->vnodeGid[i].publicIp = pDnode->publicIp; + pVgroup->vnodeGid[i].vnode = pVgroup->vgId; + } - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb == NULL) continue; + mgmtAddVgroupIntoDb(pVgroup); - pVgroup->prev = NULL; - pVgroup->next = NULL; + return TSDB_CODE_SUCCESS; +} - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)malloc(size); - if (pVgroup->tableList == NULL) { - mError("failed to malloc(size:%d) for the tableList of vgroups", size); - return -1; - } - - memset(pVgroup->tableList, 0, size); +static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + + if (pVgroup->pDb != NULL) { + mgmtRemoveVgroupFromDb(pVgroup); + } - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); - if (pVgroup->idPool == NULL) { - mError("failed to taosInitIdPool for vgroups"); - free(pVgroup->tableList); - return -1; - } - - taosIdPoolReinit(pVgroup->idPool); + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - if (tsIsCluster && pVgroup->vnodeGid[0].publicIp == 0) { - pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp); - pVgroup->vnodeGid[0].privateIp = inet_addr(tsPrivateIp); - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); + SDbObj *pDb = pVgroup->pDb; + if (pDb != NULL) { + if (pDb->cfg.maxSessions != oldTables) { + mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); + taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); + int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxSessions; + pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size); } + } + + mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxSessions, pVgroup->numOfVnodes); + return TSDB_CODE_SUCCESS; +} - // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); +static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + if (pOper->maxRowSize < tsVgUpdateSize) { + return -1; + } else { + memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); + pOper->rowSize = tsVgUpdateSize; + return TSDB_CODE_SUCCESS; + } +} + +static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { + SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj)); + if (pVgroup == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; + + memcpy(pVgroup, pOper->rowData, tsVgUpdateSize); + pOper->pObj = pVgroup; + return TSDB_CODE_SUCCESS; +} + +int32_t mgmtInitVgroups() { + SVgObj tObj; + tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "vgroups", + .hashSessions = TSDB_MAX_VGROUPS, + .maxRowSize = tsVgUpdateSize, + .keyType = SDB_KEY_TYPE_AUTO, + .insertFp = mgmtVgroupActionInsert, + .deleteFp = mgmtVgroupActionDelete, + .updateFp = mgmtVgroupActionUpdate, + .encodeFp = mgmtVgroupActionEncode, + .decodeFp = mgmtVgroupActionDecode, + .destroyFp = mgmtVgroupActionDestroy, + }; + + tsVgroupSdb = sdbOpenTable(&tableDesc); + if (tsVgroupSdb == NULL) { + mError("failed to init vgroups data"); + return -1; } mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mgmtGetVgroupMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mgmtRetrieveVgroups); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mgmtProcessCreateVnodeRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mgmtProcessDropVnodeRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg); mTrace("vgroup is initialized"); return 0; @@ -139,32 +188,41 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg) { if (pDb == NULL) { mError("failed to create vgroup, db not found"); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB); + mgmtFreeQueuedMsg(pMsg); return; } - SVgObj *pVgroup = (SVgObj *)calloc(sizeof(SVgObj), 1); + SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); strcpy(pVgroup->dbName, pDb->name); pVgroup->numOfVnodes = pDb->cfg.replications; + pVgroup->createdTime = taosGetTimestampMs(); if (mgmtAllocVnodes(pVgroup) != 0) { mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes); free(pVgroup); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES); + mgmtFreeQueuedMsg(pMsg); return; } - pVgroup->createdTime = taosGetTimestampMs(); - pVgroup->tableList = (STableInfo **) calloc(sizeof(STableInfo *), pDb->cfg.maxSessions); - pVgroup->numOfTables = 0; - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); - - mgmtAddVgroupIntoDb(pDb, pVgroup); - // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj) + }; - sdbInsertRow(tsVgroupSdb, pVgroup, 0); + int32_t code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pVgroup); + code = TSDB_CODE_SDB_ERROR; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR); + mgmtFreeQueuedMsg(pMsg); + return; + } mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - mPrint("vgroup:%d, dnode:%d vnode:%d", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode); + mPrint("vgroup:%d, index:%d, dnode:%d vnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode); } pMsg->ahandle = pVgroup; @@ -178,28 +236,12 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) { } else { mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mgmtSendDropVgroupMsg(pVgroup, NULL); - sdbDeleteRow(tsVgroupSdb, pVgroup); - } -} - -void mgmtSetVgroupIdPool() { - void * pNode = NULL; - SVgObj *pVgroup = NULL; - SDbObj *pDb; - - while (1) { - pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); - if (pVgroup == NULL || pVgroup->idPool == 0) break; - - taosIdPoolSetFreeList(pVgroup->idPool); - pVgroup->numOfTables = taosIdPoolNumOfUsed(pVgroup->idPool); - - pDb = mgmtGetDb(pVgroup->dbName); - pDb->numOfTables += pVgroup->numOfTables; - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1) - mgmtAddVgroupIntoDbTail(pDb, pVgroup); - else - mgmtAddVgroupIntoDb(pDb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + sdbDeleteRow(&oper); } } @@ -230,15 +272,15 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vgroup_status"); + strcpy(pSchema[cols].name, "vgroup status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; int32_t maxReplica = 0; SVgObj *pVgroup = NULL; - STableInfo *pTable = NULL; + SChildTableObj *pTable = NULL; if (pShow->payloadLen > 0 ) { - pTable = mgmtGetTable(pShow->payload); + pTable = mgmtGetChildTable(pShow->payload); if (NULL == pTable) { return TSDB_CODE_INVALID_TABLE_ID; } @@ -270,13 +312,13 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vnode_status"); + strcpy(pSchema[cols].name, "vnode status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 16; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "public_ip"); + strcpy(pSchema[cols].name, "public ip"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } @@ -388,109 +430,30 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo return numOfRows; } -static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = row; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - pVgroup->vnodeGid[i].vnode = pVgroup->vgId; - } - - return NULL; -} - -static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = row; - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - - if (pDb != NULL) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - } - - // mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes); - tfree(pVgroup->tableList); - - return NULL; -} - -static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - mgmtVgroupActionReset(row, str, size, ssize); - - SVgObj *pVgroup = (SVgObj *) row; - int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb != NULL) { - if (pDb->cfg.maxSessions != oldTables) { - mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); - taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size); - } - } - - mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); - - return NULL; -} - -static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - if (size < tsVgUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pVgroup, tsVgUpdateSize); - *ssize = tsVgUpdateSize; +void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { + if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] == NULL) { + pVgroup->tableList[pTable->sid] = pTable; + taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables++; } - return NULL; -} - -static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) malloc(sizeof(SVgObj)); - if (pVgroup == NULL) return NULL; - memset(pVgroup, 0, sizeof(SVgObj)); - - int32_t tsVgUpdateSize = pVgroup->updateEnd - (int8_t *) pVgroup; - memcpy(pVgroup, str, tsVgUpdateSize); - - return (void *) pVgroup; + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + mgmtAddVgroupIntoDbTail(pVgroup); } -static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - memcpy(pVgroup, str, tsVgUpdateSize); - return NULL; -} - -static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - if (pVgroup->idPool) { - taosIdPoolCleanUp(pVgroup->idPool); - pVgroup->idPool = NULL; +void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { + if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] != NULL) { + pVgroup->tableList[pTable->sid] = NULL; + taosFreeId(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables--; } - if (pVgroup->tableList) tfree(pVgroup->tableList); - tfree(row); - return NULL; -} -void mgmtUpdateVgroup(SVgObj *pVgroup) { - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 0); -} - -void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { - pVgroup->numOfTables++; - if (pTable->sid >= 0) - pVgroup->tableList[pTable->sid] = pTable; -} - -void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { - pVgroup->numOfTables--; - if (pTable->sid >= 0) - pVgroup->tableList[pTable->sid] = NULL; - taosFreeId(pVgroup->idPool, pTable->sid); + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + mgmtAddVgroupIntoDbTail(pVgroup); } SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + SDbObj *pDb = pVgroup->pDb; if (pDb == NULL) return NULL; SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg)); @@ -514,26 +477,13 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SVnodeDesc *vpeerDesc = pVnode->vpeerDesc; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { + vpeerDesc[j].vnode = htonl(pVgroup->vnodeGid[j].vnode); vpeerDesc[j].ip = htonl(pVgroup->vnodeGid[j].privateIp); } return pVnode; } -SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode) { - if (vnode < 0 || vnode >= TSDB_MAX_VNODES) { - return NULL; - } - - SDnodeObj *pDnode = mgmtGetDnode(dnode); - if (pDnode == NULL) { - return NULL; - } - - int32_t vgId = pDnode->vload[vnode].vgId; - return mgmtGetVgroup(vgId); -} - SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup) { SRpcIpSet ipSet = { .numOfIps = pVgroup->numOfVnodes, @@ -603,13 +553,23 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { newMsg->contLen = queueMsg->contLen; newMsg->pCont = rpcMallocCont(newMsg->contLen); memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); + queueMsg->pCont = NULL; mgmtAddToShellQueue(newMsg); } else { - sdbDeleteRow(tsVgroupSdb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + int32_t code = sdbDeleteRow(&oper); + if (code != 0) { + code = TSDB_CODE_SDB_ERROR; + } + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); } - free(queueMsg); + mgmtFreeQueuedMsg(queueMsg); } static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) { @@ -659,7 +619,15 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (queueMsg->received != queueMsg->expected) return; - sdbDeleteRow(tsVgroupSdb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + int32_t code = sdbDeleteRow(&oper); + if (code != 0) { + code = TSDB_CODE_SDB_ERROR; + } SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); newMsg->msgType = queueMsg->msgType; @@ -671,50 +639,71 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); mgmtAddToShellQueue(newMsg); - free(queueMsg); + queueMsg->pCont = NULL; + mgmtFreeQueuedMsg(queueMsg); } -void mgmtUpdateVgroupIp(SDnodeObj *pDnode) { - void * pNode = NULL; +static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont; + pCfg->dnodeId = htonl(pCfg->dnodeId); + pCfg->vgId = htonl(pCfg->vgId); + + SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnodeId); + if (pDnode == NULL) { + mTrace("dnode:%s, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); + return; + } + + SVgObj *pVgroup = mgmtGetVgroup(pCfg->vgId); + if (pVgroup == NULL) { + mTrace("dnode:%s, vgId:%d, no vgroup info", taosIpStr(pCfg->dnodeId), pCfg->vgId); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp); + mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL); +} + +void mgmtDropAllVgroups(SDbObj *pDropDb) { + void *pNode = NULL; + void *pLastNode = NULL; + int32_t numOfTables = 0; + int32_t dbNameLen = strlen(pDropDb->name); SVgObj *pVgroup = NULL; + while (1) { pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); if (pVgroup == NULL) break; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *vnodeGid = pVgroup->vnodeGid + i; - if (vnodeGid->dnodeId == pDnode->dnodeId) { - mPrint("vgroup:%d, dnode:%d, privateIp:%s change to %s, publicIp:%s change to %s", - pVgroup->vgId, vnodeGid->dnodeId, pDnode->privateIp, taosIpStr(vnodeGid->privateIp), - pDnode->publicIp, taosIpStr(vnodeGid->publicIp)); - vnodeGid->publicIp = pDnode->publicIp; - vnodeGid->privateIp = pDnode->privateIp; - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); - } + if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + }; + sdbDeleteRow(&oper); + pNode = pLastNode; + numOfTables++; + continue; } } + + mTrace("db:%s, all vgroups is dropped from sdb", pDropDb->name, numOfTables); } -//static void mgmtProcessVnodeCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *pConn) { -// if (!sdbMaster) { -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0); -// return; -// } -// -// SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) pCont; -// pCfg->dnode = htonl(pCfg->dnode); -// pCfg->vnode = htonl(pCfg->vnode); -// -// SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode); -// if (pVgroup == NULL) { -// mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_NOT_ACTIVE_VNODE, NULL, 0); -// return; -// } -// -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0); -// -// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); -// mgmtSendCreateVnodeMsg(pVgroup, pCfg->vnode, &ipSet, NULL); -//} -// \ No newline at end of file +void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { + assert(ahandle != NULL); + + if (pVgroup->numOfVnodes != pVgroup->pDb->cfg.replications) { + // TODO: + // mgmtSendAlterVgroupMsg(pVgroup, NULL); + } else { + mgmtAddToShellQueue(ahandle); + } +} \ No newline at end of file diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index 0e51962f49999ae55c1c18aa5acd9d9baba64be7..a6462f98558d163b05d3b8b1cbc2cad23ae7cae4 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -12,4 +12,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(query ${SRC}) TARGET_LINK_LIBRARIES(query tsdb tutil m rt) -ENDIF () \ No newline at end of file +ENDIF () + +ADD_SUBDIRECTORY(tests) \ No newline at end of file diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 1e89ffb49dbc0667dd25d7d8b32646a155bcb016..d52eb24726891a0beb3e4b3aa9c2a061cf902d80 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include #include "os.h" #include "taosmsg.h" @@ -93,8 +94,7 @@ void tSQLBinaryExprToString(tSQLBinaryExpr *pExpr, char *dst, int32_t *len); void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExprs, void (*fp)(void*)); -void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExprs, struct tSkipList *pSkipList, tQueryResultset *result, - SBinaryFilterSupp *param); +void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, char *, int32_t)); diff --git a/src/query/inc/qsqltype.h b/src/query/inc/qsqltype.h index 5d85ab4b80ebad4534ea6fa09b23d3ed97df04c0..ad9affa1ccdc3a7d729646ba1c0bf67063273ad1 100644 --- a/src/query/inc/qsqltype.h +++ b/src/query/inc/qsqltype.h @@ -54,7 +54,7 @@ enum _sql_type { TSDB_SQL_CONNECT, TSDB_SQL_USE_DB, // 30 TSDB_SQL_META, - TSDB_SQL_METRIC, + TSDB_SQL_STABLEVGROUP, TSDB_SQL_MULTI_META, TSDB_SQL_HB, diff --git a/src/query/inc/qtsbuf.h b/src/query/inc/qtsbuf.h index 1afdb0cd6cb9175b14675d06446afbe3a891df27..8e014e5feb781b2c87209118d1c4c8a7c2d13813 100644 --- a/src/query/inc/qtsbuf.h +++ b/src/query/inc/qtsbuf.h @@ -48,10 +48,10 @@ typedef struct STSElem { } STSElem; typedef struct STSCursor { - int32_t vnodeIndex; - int32_t blockIndex; - int32_t tsIndex; - int32_t order; + int32_t vnodeIndex; + int32_t blockIndex; + int32_t tsIndex; + uint32_t order; } STSCursor; typedef struct STSBlock { diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 8956fb52b13e557ec5f1d36a7210cacff72a35e8..5adce04efa44d7e85bc9ccf482dd21e0deff20b5 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -32,12 +32,6 @@ typedef struct SData { char data[]; } SData; -enum { - ST_QUERY_KILLED = 0, // query killed - ST_QUERY_PAUSED = 1, // query paused, due to full of the response buffer - ST_QUERY_COMPLETED = 2, // query completed -}; - struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); @@ -60,16 +54,20 @@ typedef struct SWindowStatus { } SWindowStatus; typedef struct SWindowResult { - uint16_t numOfRows; + uint16_t numOfRows; // number of rows of current time window SPosInfo pos; // Position of current result in disk-based output buffer SResultInfo* resultInfo; // For each result column, there is a resultInfo STimeWindow window; // The time window that current result covers. - SWindowStatus status; + SWindowStatus status; // this result status: closed or opened } SWindowResult; typedef struct SResultRec { - int64_t pointsTotal; - int64_t pointsRead; + int64_t total; // total generated result size in rows + int64_t size; // current result set size in rows + int64_t capacity; // capacity of current result output buffer + + // result size threshold in rows. If the result buffer is larger than this, pause query and return to client + int32_t threshold; } SResultRec; typedef struct SWindowResInfo { @@ -97,7 +95,6 @@ typedef struct SSingleColumnFilterInfo { void* pData; } SSingleColumnFilterInfo; -/* intermediate pos during multimeter query involves interval */ typedef struct STableQueryInfo { int64_t lastKey; STimeWindow win; @@ -105,16 +102,15 @@ typedef struct STableQueryInfo { int16_t queryRangeSet; // denote if the query range is set, only available for interval query int64_t tag; STSCursor cur; - int32_t sid; // for retrieve the page id list + int32_t tid; // for retrieve the page id list SWindowResInfo windowResInfo; } STableQueryInfo; typedef struct STableDataInfo { int32_t numOfBlocks; - int32_t start; // start block index + int32_t start; // start block index int32_t tableIndex; - void* pMeterObj; int32_t groupIdx; // group id in table list STableQueryInfo* pTableQInfo; } STableDataInfo; @@ -143,7 +139,6 @@ typedef struct SQuery { int32_t pos; int64_t pointsOffset; // the number of points offset to save read data SData** sdata; - int32_t capacity; SSingleColumnFilterInfo* pFilterInfo; } SQuery; @@ -170,20 +165,16 @@ typedef struct SQueryRuntimeEnv { } SQueryRuntimeEnv; typedef struct SQInfo { - uint64_t signature; + void* signature; TSKEY startTime; - int64_t elapsedTime; - SResultRec rec; - int32_t pointsReturned; + TSKEY elapsedTime; int32_t pointsInterpo; - int32_t code; // error code to returned to client - int32_t killed; // denotes if current query is killed + int32_t code; // error code to returned to client sem_t dataReady; - SArray* pTableIdList; // table list + SArray* pTableIdList; // table id list SQueryRuntimeEnv runtimeEnv; int32_t subgroupIdx; int32_t offset; /* offset in group result set of subgroup */ -// tSidSet* pSidSet; T_REF_DECLARE() /* @@ -205,7 +196,7 @@ typedef struct SQInfo { * @param pQInfo * @return */ -int32_t qCreateQueryInfo(SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo); +int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo); /** * query on single table @@ -214,18 +205,24 @@ int32_t qCreateQueryInfo(SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo); void qTableQuery(SQInfo* pQInfo); /** - * query on super table - * @param pReadMsg + * wait for the query completed, and retrieve final results to client + * @param pQInfo */ -void qSuperTableQuery(void* pReadMsg); +int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo); /** - * wait for the query completed, and retrieve final results to client + * * @param pQInfo + * @param pRsp + * @return */ -int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo, int32_t *numOfRows, int32_t* rowsize); - +int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen); -//int32_t qBuildQueryResult(SQInfo* pQInfo, void* pBuf); +/** + * + * @param pQInfo + * @return + */ +bool qHasMoreResultsToRetrieve(SQInfo* pQInfo); #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 04ff9e56f8238ca393ed50a4e975191d158517dd..c7aa57200cff7e7fb6091e55dd4366875f6e57b7 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -219,7 +219,7 @@ typedef struct SQLAggFuncElem { void (*distSecondaryMergeFunc)(SQLFunctionCtx *pCtx); - int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus); + int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); } SQLAggFuncElem; typedef struct SPatternCompareInfo { diff --git a/src/query/src/qast.c b/src/query/src/qast.c index a0cbf121691c5e341964091038d233b40f1c3c58..c6c65eba27e0e96a7ca895a1cdeaeec6fd302029 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -14,6 +14,9 @@ */ #include "qast.h" +#include +#include +#include "../../client/inc/tschemautil.h" #include "os.h" #include "qsqlparser.h" #include "qsyntaxtreefunction.h" @@ -105,7 +108,7 @@ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols, return NULL; } - size_t nodeSize = sizeof(tSQLSyntaxNode); + size_t nodeSize = sizeof(tSQLSyntaxNode); tSQLSyntaxNode *pNode = NULL; if (pToken->type == TK_ID || pToken->type == TK_TBNAME) { @@ -237,9 +240,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tSQLSyntaxNode } static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { - SSQLToken t0; - - t0 = tStrGetToken(str, i, false, 0, NULL); + SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0) { return NULL; } @@ -341,7 +342,8 @@ void tSQLBinaryExprFromString(tSQLBinaryExpr **pExpr, SSchema *pSchema, int32_t return; } - int32_t pos = 0; + int32_t pos = 0; + tSQLSyntaxNode *pStxNode = createSyntaxTree(pSchema, numOfCols, src, &pos); if (pStxNode != NULL) { assert(pStxNode->nodeType == TSQL_NODE_EXPR); @@ -705,126 +707,120 @@ static bool filterItem(tSQLBinaryExpr *pExpr, const void *pItem, SBinaryFilterSu * @param pSchema tag schemas * @param fp filter callback function */ -static UNUSED_FUNC void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, tQueryResultset *pResult, SBinaryFilterSupp *param) { - int32_t n = 0; - for (int32_t i = 0; i < pResult->num; ++i) { - void *pItem = pResult->pRes[i]; +static void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, SArray *pResult, SBinaryFilterSupp *param) { + size_t size = taosArrayGetSize(pResult); + + SArray* array = taosArrayInit(size, POINTER_BYTES); + for (int32_t i = 0; i < size; ++i) { + void *pItem = taosArrayGetP(pResult, i); if (filterItem(pExpr, pItem, param)) { - pResult->pRes[n++] = pResult->pRes[i]; + taosArrayPush(array, &pItem); } } - - pResult->num = n; + + taosArrayCopy(pResult, array); } -//static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, tQueryResultset *pResult, tSkipList *pSkipList, -// SBinaryFilterSupp *param) { -// int32_t n = 0; -// SSkipListIterator iter = {0}; -// -// int32_t ret = tSkipListIteratorReset(pSkipList, &iter); -// assert(ret == 0); -// -// pResult->pRes = calloc(pSkipList->nSize, POINTER_BYTES); -// -// while (tSkipListIteratorNext(&iter)) { -// tSkipListNode *pNode = tSkipListIteratorGet(&iter); -// if (filterItem(pExpr, pNode, param)) { -// pResult->pRes[n++] = pNode; -// } -// } -// -// pResult->num = n; -//} +static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, SArray *pResult, SSkipList *pSkipList, + SBinaryFilterSupp *param) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + if (filterItem(pExpr, pNode, param)) { + taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); + } + } +} // post-root order traverse syntax tree -//void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, tSkipList *pSkipList, tQueryResultset *result, -// SBinaryFilterSupp *param) { -// if (pExpr == NULL) { -// return; -// } -// -// tSQLSyntaxNode *pLeft = pExpr->pLeft; -// tSQLSyntaxNode *pRight = pExpr->pRight; -// -// // recursive traverse left child branch -// if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { -// uint8_t weight = pLeft->pExpr->filterOnPrimaryKey + pRight->pExpr->filterOnPrimaryKey; -// -// if (weight == 0 && result->num > 0 && pSkipList == NULL) { -// /** -// * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. -// * Since no index presented, the filter operation is done by scan all elements in the result set. -// * -// * if the query is a high selectivity filter, only small portion of meters are retrieved. -// */ -// tSQLBinaryTraverseOnResult(pExpr, result, param); -// } else if (weight == 0) { -// /** -// * apply the hierarchical expression to every node in skiplist for find the qualified nodes -// */ -// assert(result->num == 0); -// tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); -// } else if (weight == 2 || (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_OR)) { -// tQueryResultset rLeft = {0}; -// tQueryResultset rRight = {0}; -// -// tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, &rLeft, param); -// tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, &rRight, param); -// -// if (pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS -// intersect(&rLeft, &rRight, result); -// } else if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or -// merge(&rLeft, &rRight, result); -// } else { -// assert(false); -// } -// -// free(rLeft.pRes); -// free(rRight.pRes); -// } else { -// /* -// * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here -// * -// * first, we filter results based on the skiplist index, which is the initial filter stage, -// * then, we conduct the secondary filter operation based on the result from the initial filter stage. -// */ -// assert(pExpr->nSQLBinaryOptr == TSDB_RELATION_AND); -// -// tSQLBinaryExpr *pFirst = NULL; -// tSQLBinaryExpr *pSecond = NULL; -// if (pLeft->pExpr->filterOnPrimaryKey == 1) { -// pFirst = pLeft->pExpr; -// pSecond = pRight->pExpr; -// } else { -// pFirst = pRight->pExpr; -// pSecond = pLeft->pExpr; -// } -// -// assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); -// -// // we filter the result based on the skiplist index in the first place -// tSQLBinaryExprTraverse(pFirst, pSkipList, result, param); -// -// /* -// * recursively perform the filter operation based on the initial results, -// * So, we do not set the skiplist index as a parameter -// */ -// tSQLBinaryExprTraverse(pSecond, NULL, result, param); -// } -// } else { // column project -// assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); -// -// param->setupInfoFn(pExpr, param->pExtInfo); -// if (pSkipList == NULL) { -// tSQLListTraverseOnResult(pExpr, param->fp, result); -// } else { +void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { + if (pExpr == NULL) { + return; + } + + tSQLSyntaxNode *pLeft = pExpr->pLeft; + tSQLSyntaxNode *pRight = pExpr->pRight; + + // recursive traverse left child branch + if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { + uint8_t weight = pLeft->pExpr->filterOnPrimaryKey + pRight->pExpr->filterOnPrimaryKey; + + if (weight == 0 && taosArrayGetSize(result) > 0 && pSkipList == NULL) { + /** + * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. + * Since no index presented, the filter operation is done by scan all elements in the result set. + * + * if the query is a high selectivity filter, only small portion of meters are retrieved. + */ + tSQLBinaryTraverseOnResult(pExpr, result, param); + } else if (weight == 0) { + /** + * apply the hierarchical expression to every node in skiplist for find the qualified nodes + */ + assert(taosArrayGetSize(result) == 0); + tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); + } else if (weight == 2 || (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_OR)) { + tQueryResultset rLeft = {0}; + tQueryResultset rRight = {0}; + + tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, &rLeft, param); + tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, &rRight, param); + + if (pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS + intersect(&rLeft, &rRight, result); + } else if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or + merge(&rLeft, &rRight, result); + } else { + assert(false); + } + + free(rLeft.pRes); + free(rRight.pRes); + } else { + /* + * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here + * + * first, we filter results based on the skiplist index, which is the initial filter stage, + * then, we conduct the secondary filter operation based on the result from the initial filter stage. + */ + assert(pExpr->nSQLBinaryOptr == TSDB_RELATION_AND); + + tSQLBinaryExpr *pFirst = NULL; + tSQLBinaryExpr *pSecond = NULL; + if (pLeft->pExpr->filterOnPrimaryKey == 1) { + pFirst = pLeft->pExpr; + pSecond = pRight->pExpr; + } else { + pFirst = pRight->pExpr; + pSecond = pLeft->pExpr; + } + + assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); + + // we filter the result based on the skiplist index in the first place + tSQLBinaryExprTraverse(pFirst, pSkipList, result, param); + + /* + * recursively perform the filter operation based on the initial results, + * So, we do not set the skiplist index as a parameter + */ + tSQLBinaryExprTraverse(pSecond, NULL, result, param); + } + } else { // column project + assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); + + param->setupInfoFn(pExpr, param->pExtInfo); + if (pSkipList == NULL) { + tSQLListTraverseOnResult(pExpr, param->fp, result); + } else { // assert(result->num == 0); -//// tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result); -// } -// } -//} +// tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result); + } + } +} void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, char *, int32_t)) { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 0a5abccdd576710f69419298bc619a806efab143..83cbae42664407d1245adfc9740fa31be5af1de0 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -14,9 +14,9 @@ */ #include "os.h" -#include "taosmsg.h" #include "hash.h" #include "hashfunc.h" +#include "taosmsg.h" #include "tlog.h" #include "tlosertree.h" #include "tscompression.h" @@ -53,9 +53,9 @@ /* get the qinfo struct address from the query struct address */ #define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].info.bytes) + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdx].info.bytes) #define GET_COLUMN_TYPE(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].info.type) + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdx].info.type) typedef struct SPointInterpoSupporter { int32_t numOfCols; @@ -64,38 +64,24 @@ typedef struct SPointInterpoSupporter { } SPointInterpoSupporter; typedef enum { - - /* - * the program will call this function again, if this status is set. - * used to transfer from QUERY_RESBUF_FULL - */ + // when query starts to execute, this status will set QUERY_NOT_COMPLETED = 0x1u, - /* - * output buffer is full, so, the next query will be employed, - * in this case, we need to set the appropriated start scan point for - * the next query. - * - * this status is only exist in group-by clause and - * diff/add/division/multiply/ query. + /* result output buffer is full, current query is paused. + * this status is only exist in group-by clause and diff/add/division/multiply/ query. */ QUERY_RESBUF_FULL = 0x2u, - /* - * query is over - * 1. this status is used in one row result query process, e.g., - * count/sum/first/last/ - * avg...etc. - * 2. when the query range on timestamp is satisfied, it is also denoted as - * query_compeleted + /* query is over + * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. + * 2. when all data within queried time window, it is also denoted as query_completed */ QUERY_COMPLETED = 0x4u, - - /* - * all data has been scanned, so current search is stopped, - * At last, the function will transfer this status to QUERY_COMPLETED + + /* when the result is not completed return to client, this status will be + * usually used in case of interval query with interpolation option */ - QUERY_NO_DATA_TO_CHECK = 0x8u, + QUERY_OVER = 0x8u, } vnodeQueryStatus; static void setQueryStatus(SQuery *pQuery, int8_t status); @@ -119,7 +105,7 @@ static int32_t flushFromResultBuf(SQInfo *pQInfo); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); -static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char *primaryColumnData, int32_t size, +static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag); static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); @@ -378,8 +364,8 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functio bool doRevisedResultsByLimit(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - if ((pQuery->limit.limit > 0) && (pQuery->rec.pointsRead + pQInfo->rec.pointsRead > pQuery->limit.limit)) { - pQuery->rec.pointsRead = pQuery->limit.limit - pQInfo->rec.pointsRead; + if ((pQuery->limit.limit > 0) && (pQuery->rec.size + pQuery->rec.size > pQuery->limit.limit)) { + pQuery->rec.size = pQuery->limit.limit - pQuery->rec.size; // query completed setQueryStatus(pQuery, QUERY_COMPLETED); @@ -389,29 +375,6 @@ bool doRevisedResultsByLimit(SQInfo *pQInfo) { return false; } -/** - * - * @param pQuery - * @param pDataBlockInfo - * @param forwardStep - * @return TRUE means query not completed, FALSE means query is completed - */ -static bool queryPaused(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, int32_t forwardStep) { - // output buffer is full, pause current query - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - // assert((QUERY_IS_ASC_QUERY(pQuery) && forwardStep + pQuery->pos <= pDataBlockInfo->size) || - // (!QUERY_IS_ASC_QUERY(pQuery) && pQuery->pos - forwardStep + 1 >= 0)); - // - return true; - } - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - return true; - } - - return false; -} - static bool isTopBottomQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; @@ -427,15 +390,9 @@ static bool isTopBottomQuery(SQuery *pQuery) { return false; } -static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - int32_t columnIndex) { - // no SField info exist, or column index larger than the output column, no result. - if (pStatis == NULL) { - return NULL; - } - +static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t index) { // for a tag column, no corresponding field info - SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[columnIndex].pBase.colInfo; + SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[index].pBase.colInfo; if (TSDB_COL_IS_TAG(pColIndexEx->flag)) { return NULL; } @@ -453,13 +410,31 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo return NULL; } +/** + * @param pQuery + * @param col + * @param pDataBlockInfo + * @param pStatis + * @param pColStatis + * @return + */ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SDataStatis **pColStatis) { - if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[col].pBase.colInfo.flag) || pStatis == NULL) { + SColIndexEx* pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; + if (TSDB_COL_IS_TAG(pColIndex->flag)) { return false; } - - *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); + + // query on primary timestamp column, not null value at all + if (pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return false; + } + + *pColStatis = NULL; + if (pStatis != NULL) { + *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); + } + if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { return false; } @@ -887,7 +862,6 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 * the remain meter may not have the required column in cache actually. * So, the validation of required column in cache with the corresponding meter schema is reinforced. */ - if (pDataBlock == NULL) { return NULL; } @@ -939,10 +913,11 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; SDataStatis *tpField = NULL; - bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); - char * dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + + bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); + char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, (char *)primaryKeyCol, pDataBlockInfo->size, functionId, tpField, + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->size, functionId, tpField, hasNull, &sasArray[k], pRuntimeEnv->scanFlag); } @@ -1289,7 +1264,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat if (pRuntimeEnv->pTSBuf != NULL) { // if timestamp filter list is empty, quit current query if (!tsBufNextPos(pRuntimeEnv->pTSBuf)) { - setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + setQueryStatus(pQuery, QUERY_COMPLETED); break; } } @@ -1346,17 +1321,16 @@ static int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int32_t forward static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, __block_search_fn_t searchFn, int32_t *numOfRes, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { - SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + SQuery *pQuery = pRuntimeEnv->pQuery; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { *numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { *numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - + TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; - pQuery->lastKey = lastKey + step; + pQuery->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); @@ -1370,18 +1344,14 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl assert(*numOfRes >= 0); // check if buffer is large enough for accommodating all qualified points - if (*numOfRes > 0 && pQuery->checkBufferInLoop == 1) { - pQuery->pointsOffset -= *numOfRes; - if (pQuery->pointsOffset <= 0) { // todo return correct numOfRes for ts_comp function - pQuery->pointsOffset = 0; - setQueryStatus(pQuery, QUERY_RESBUF_FULL); - } + if (*numOfRes > 0 && pQuery->checkBufferInLoop == 1 && ((*numOfRes) >= pQuery->rec.threshold)) { + setQueryStatus(pQuery, QUERY_RESBUF_FULL); } return 0; } -void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char *primaryColumnData, int32_t size, +void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag) { pCtx->scanFlag = scanFlag; @@ -1396,15 +1366,15 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char * pCtx->preAggVals.isSet = false; } - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (primaryColumnData != NULL)) { - pCtx->ptsList = (int64_t *)(primaryColumnData); + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (tsCol != NULL)) { + pCtx->ptsList = tsCol; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { // last_dist or first_dist function // store the first&last timestamp into the intermediate buffer [1], the true // value may be null but timestamp will never be null - pCtx->ptsList = (int64_t *)(primaryColumnData); + pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { /* @@ -1420,7 +1390,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char * pTWAInfo->EKey = pQuery->window.ekey; } - pCtx->ptsList = (int64_t *)(primaryColumnData); + pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_ARITHM) { pCtx->param[1].pz = param; @@ -1482,10 +1452,11 @@ static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool i } } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order, bool isSTableQuery) { +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order, + bool isSTableQuery) { dTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); - SQuery* pQuery = pRuntimeEnv->pQuery; - + SQuery *pQuery = pRuntimeEnv->pQuery; + pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutputCols, sizeof(SQLFunctionCtx)); @@ -1499,16 +1470,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel SColIndexEx * pColIndexEx = &pSqlFuncMsg->colInfo; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - - if (TSDB_COL_IS_TAG(pSqlFuncMsg->colInfo.flag)) { // process tag column info - SSchema *pSchema = getColumnModelSchema(pTagsSchema, pColIndexEx->colIdx); - - pCtx->inputType = pSchema->type; - pCtx->inputBytes = pSchema->bytes; - } else { - pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); - pCtx->inputBytes = GET_COLUMN_BYTES(pQuery, i); - } + pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); + pCtx->inputBytes = GET_COLUMN_BYTES(pQuery, i); pCtx->ptsOutputBuf = NULL; @@ -1608,10 +1571,8 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); } -bool isQueryKilled(SQuery *pQuery) { - return false; - - SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); +static bool isQueryKilled(SQInfo *pQInfo) { + return (pQInfo->code == TSDB_CODE_QUERY_CANCELLED); #if 0 /* * check if the queried meter is going to be deleted. @@ -1625,7 +1586,10 @@ bool isQueryKilled(SQuery *pQuery) { return (pQInfo->killed == 1); #endif - return 0; +} + +static bool setQueryKilled(SQInfo* pQInfo) { + pQInfo->code = TSDB_CODE_QUERY_CANCELLED; } bool isFixedOutputQuery(SQuery *pQuery) { @@ -1703,7 +1667,7 @@ bool notHasQueryTimeRange(SQuery *pQuery) { (pQuery->window.skey == INT64_MAX && pQuery->window.ekey == 0 && (!QUERY_IS_ASC_QUERY(pQuery))); } -bool needSupplementaryScan(SQuery *pQuery) { +static bool needReverseScan(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { @@ -1890,8 +1854,6 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { pQuery->checkBufferInLoop = hasMultioutput ? 1 : 0; } - - // pQuery->pointsOffset = pQuery->pointsToRead; } /* @@ -2242,7 +2204,7 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { size_t s = taosArrayGetSize(pQInfo->pTableIdList); num = MAX(s, INITIAL_RESULT_ROWS_VALUE); } else { // for super table query, one page for each subset -// num = pQInfo->pSidSet->numOfSubSet; + num = 1;//pQInfo->pSidSet->numOfSubSet; } assert(num > 0); @@ -2289,7 +2251,7 @@ void vnodeQueryFreeQInfoEx(SQInfo *pQInfo) { // tSidSetDestroy(&pQInfo->pSidSet); if (pQInfo->pTableDataInfo != NULL) { -// size_t num = taosHashGetSize(pQInfo->pTableIdList); + // size_t num = taosHashGetSize(pQInfo->pTableIdList); for (int32_t j = 0; j < 0; ++j) { destroyMeterQueryInfo(pQInfo->pTableDataInfo[j].pTableQInfo, pQuery->numOfOutputCols); } @@ -2312,7 +2274,7 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { pQuery->status = 0; - pQInfo->rec = (SResultRec){0}; + pQuery->rec = (SResultRec){0}; pQuery->rec = (SResultRec){0}; changeExecuteScanOrder(pQuery, true); @@ -2328,7 +2290,7 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { pQuery->lastKey = pQuery->window.skey; // create runtime environment -// SColumnModel *pTagSchemaInfo = pQInfo->pSidSet->pColumnModel; + // SColumnModel *pTagSchemaInfo = pQInfo->pSidSet->pColumnModel; // get one queried meter assert(0); @@ -2388,7 +2350,7 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { taosArrayPush(cols, &pQuery->colList[i]); } - pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(&cond, sa, cols); + pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(NULL, &cond, sa, cols); // metric query do not invoke interpolation, it will be done at the second-stage merge if (!isPointInterpoQuery(pQuery)) { @@ -2409,7 +2371,7 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { */ void vnodeDecMeterRefcnt(SQInfo *pQInfo) { if (pQInfo != NULL) { -// assert(taosHashGetSize(pQInfo->pTableIdList) >= 1); + // assert(taosHashGetSize(pQInfo->pTableIdList) >= 1); } #if 0 @@ -2523,22 +2485,18 @@ static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow) { } SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo, SDataStatis **pStatis) { - SQuery * pQuery = pRuntimeEnv->pQuery; - tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; + SQuery *pQuery = pRuntimeEnv->pQuery; uint32_t r = 0; SArray * pDataBlock = NULL; - // STimeWindow *w = &pQueryHandle->window; - if (pQuery->numOfFilterCols > 0) { r = BLK_DATA_ALL_NEEDED; } else { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; int32_t colId = pQuery->pSelectExpr[i].pBase.colInfo.colId; - - // r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], w->skey, w->ekey, colId); + r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId); } if (pRuntimeEnv->pTSBuf > 0 || isIntervalQuery(pQuery)) { @@ -2552,10 +2510,10 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl // pBlock->keyFirst, pBlock->keyLast, pBlock->numOfPoints); } else if (r == BLK_DATA_FILEDS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { - // return DISK_DATA_LOAD_FAILED; +// return DISK_DATA_LOAD_FAILED; } - if (pStatis == NULL) { + if (*pStatis == NULL) { pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); } } else { @@ -2587,23 +2545,23 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; TSKEY *keyList; - + if (num <= 0) return -1; - + keyList = (TSKEY *)pValue; firstPos = 0; lastPos = num - 1; - + if (order == 0) { // find the first position which is smaller than the key while (1) { if (key >= keyList[lastPos]) return lastPos; if (key == keyList[firstPos]) return firstPos; if (key < keyList[firstPos]) return firstPos - 1; - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -2612,13 +2570,13 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { break; } } - + } else { // find the first position which is bigger than the key while (1) { if (key <= keyList[firstPos]) return firstPos; if (key == keyList[lastPos]) return lastPos; - + if (key > keyList[lastPos]) { lastPos = lastPos + 1; if (lastPos >= num) @@ -2626,10 +2584,10 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { else return lastPos; } - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -2639,7 +2597,7 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } } } - + return midPos; } @@ -2648,65 +2606,64 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { int64_t cnt = 0; dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", - GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); - + GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); + tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; - + while (tsdbNextDataBlock(pQueryHandle)) { // check if query is killed or not set the status of query to pass the status check - if (isQueryKilled(pQuery)) { - setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return cnt; } - + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - + if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) { - TSKEY skey1, ekey1; - STimeWindow w = {0}; - SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; - + TSKEY skey1, ekey1; + STimeWindow w = {0}; + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (QUERY_IS_ASC_QUERY(pQuery)) { - doGetAlignedIntervalQueryRangeImpl(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, &ekey1, &w); + doGetAlignedIntervalQueryRangeImpl(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, + &skey1, &ekey1, &w); pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp TSKEY winStart = blockInfo.window.ekey - pQuery->intervalTime; - doGetAlignedIntervalQueryRangeImpl(pQuery, winStart, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, &w); - + doGetAlignedIntervalQueryRangeImpl(pQuery, winStart, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, + &w); + pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } } - + int32_t numOfRes = 0; - SDataStatis *pStatis = NULL; SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &numOfRes, &pRuntimeEnv->windowResInfo, pDataBlock); - -// dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", fileId:%d, slot:%d, pos:%d, rows:%d, checked:%d", -// GET_QINFO_ADDR(pQuery), blockInfo.window.skey, blockInfo.window.ekey, pQueryHandle->cur.fileId, pQueryHandle->cur.slot, -// pQuery->pos, blockInfo.size, forwardStep); - + + dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d", + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.size); + // save last access position cnt += forwardStep; if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { break; } } - + // if the result buffer is not full, set the query completed flag if (!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { setQueryStatus(pQuery, QUERY_COMPLETED); } - + if (isIntervalQuery(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { - int32_t step = QUERY_IS_ASC_QUERY(pQuery)? QUERY_ASC_FORWARD_STEP:QUERY_DESC_FORWARD_STEP; - + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + int32_t step = QUERY_IS_ASC_QUERY(pQuery) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP; + closeAllTimeWindow(&pRuntimeEnv->windowResInfo); removeRedundantWindow(&pRuntimeEnv->windowResInfo, pQuery->lastKey - step, step); pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; @@ -2714,7 +2671,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); } } - + return cnt; } @@ -2969,36 +2926,37 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) return leftTimestamp > rightTimestamp ? 1 : -1; } -int32_t mergeMetersResultToOneGroups(SQInfo *pQInfo) { +int32_t mergeResultsToGroup(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; int64_t st = taosGetTimestampMs(); int32_t ret = TSDB_CODE_SUCCESS; -// while (pQInfo->subgroupIdx < pQInfo->pSidSet->numOfSubSet) { -// int32_t start = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx]; -// int32_t end = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx + 1]; -// -// assert(0); -// // ret = doMergeMetersResultsToGroupRes(pQInfo, pQuery, pRuntimeEnv, pQInfo->pTableDataInfo, start, end); -// if (ret < 0) { // not enough disk space to save the data into disk -// return -1; -// } -// -// pQInfo->subgroupIdx += 1; -// -// // this group generates at least one result, return results -// if (ret > 0) { -// break; -// } -// -// assert(pQInfo->numOfGroupResultPages == 0); -// dTrace("QInfo:%p no result in group %d, continue", GET_QINFO_ADDR(pQuery), pQInfo->subgroupIdx - 1); -// } -// -// dTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", GET_QINFO_ADDR(pQuery), -// pQInfo->subgroupIdx - 1, pQInfo->pSidSet->numOfSubSet, taosGetTimestampMs() - st); + // while (pQInfo->subgroupIdx < pQInfo->pSidSet->numOfSubSet) { + // int32_t start = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx]; + // int32_t end = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx + 1]; + // + // assert(0); + // // ret = doMergeMetersResultsToGroupRes(pQInfo, pQuery, pRuntimeEnv, pQInfo->pTableDataInfo, start, end); + // if (ret < 0) { // not enough disk space to save the data into disk + // return -1; + // } + // + // pQInfo->subgroupIdx += 1; + // + // // this group generates at least one result, return results + // if (ret > 0) { + // break; + // } + // + // assert(pQInfo->numOfGroupResultPages == 0); + // dTrace("QInfo:%p no result in group %d, continue", GET_QINFO_ADDR(pQuery), pQInfo->subgroupIdx - 1); + // } + // + // dTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", + // GET_QINFO_ADDR(pQuery), + // pQInfo->subgroupIdx - 1, pQInfo->pSidSet->numOfSubSet, taosGetTimestampMs() - st); return TSDB_CODE_SUCCESS; } @@ -3008,15 +2966,15 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { pQInfo->numOfGroupResultPages = 0; // current results of group has been sent to client, try next group - if (mergeMetersResultToOneGroups(pQInfo) != TSDB_CODE_SUCCESS) { + if (mergeResultsToGroup(pQInfo) != TSDB_CODE_SUCCESS) { return; // failed to save data in the disk } // set current query completed -// if (pQInfo->numOfGroupResultPages == 0 && pQInfo->subgroupIdx == pQInfo->pSidSet->numOfSubSet) { - // pQInfo->tableIndex = pQInfo->pSidSet->numOfTables; -// return; -// } + // if (pQInfo->numOfGroupResultPages == 0 && pQInfo->subgroupIdx == pQInfo->pSidSet->numOfSubSet) { + // pQInfo->tableIndex = pQInfo->pSidSet->numOfTables; + // return; + // } } SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; @@ -3031,7 +2989,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { total += pData->numOfElems; } - pQuery->sdata[0]->num = total; + int32_t rows = total; int32_t offset = 0; for (int32_t num = 0; num < list.size; ++num) { @@ -3039,7 +2997,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - char * pDest = pQuery->sdata[i]->data; + char * pDest = pQuery->sdata[i]; memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->numOfElems, bytes * pData->numOfElems); @@ -3048,9 +3006,9 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { offset += pData->numOfElems; } - assert(pQuery->rec.pointsRead == 0); + assert(pQuery->rec.size == 0); - pQuery->rec.pointsRead += pQuery->sdata[0]->num; + pQuery->rec.size += rows; pQInfo->offset += 1; } @@ -3089,9 +3047,9 @@ int32_t doMergeMetersResultsToGroupRes(SQInfo *pQInfo, STableDataInfo *pTableDat // todo opt for the case of one table per group int32_t numOfMeters = 0; for (int32_t i = start; i < end; ++i) { - int32_t sid = pTableDataInfo[i].pTableQInfo->sid; + int32_t tid = pTableDataInfo[i].pTableQInfo->tid; - SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, sid); + SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, tid); if (list.size > 0 && pTableDataInfo[i].pTableQInfo->windowResInfo.size > 0) { pTableList[numOfMeters] = &pTableDataInfo[i]; numOfMeters += 1; @@ -3217,7 +3175,8 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. int32_t pageId = -1; - int32_t remain = pQuery->sdata[0]->num; + assert(0); + int32_t remain = 0;//pQuery->sdata[0]->num; int32_t offset = 0; while (remain > 0) { @@ -3248,19 +3207,18 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; + pCtx[k].aOutputBuf = pQuery->sdata[k] - pCtx[k].outputBytes; pCtx[k].size = 1; pCtx[k].startOffset = 0; pCtx[k].resultInfo = &pResultInfo[k]; - pQuery->sdata[k]->num = 0; + pQuery->sdata[k] = 0; } } -void setMeterDataInfo(STableDataInfo *pTableDataInfo, void *pMeterObj, int32_t meterIdx, int32_t groupId) { - pTableDataInfo->pMeterObj = pMeterObj; +void setTableDataInfo(STableDataInfo *pTableDataInfo, int32_t tableIndex, int32_t groupId) { pTableDataInfo->groupIdx = groupId; - pTableDataInfo->tableIndex = meterIdx; + pTableDataInfo->tableIndex = tableIndex; } static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo *pWindowResInfo, int32_t order) { @@ -3314,7 +3272,7 @@ void disableFunctForTableSuppleScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order pQuery->order.order = pQuery->order.order ^ 1u; } -void disableFunctForSuppleScan(SQInfo *pQInfo, int32_t order) { +void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -3339,7 +3297,7 @@ void disableFunctForSuppleScan(SQInfo *pQInfo, int32_t order) { pQuery->order.order = (pQuery->order.order) ^ 1u; } -void enableFunctForMasterScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { +void enableFuncForForwardScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { @@ -3379,7 +3337,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * pQuery->capacity); + memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); } initCtxOutputBuf(pRuntimeEnv); @@ -3426,14 +3384,14 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - if (pQuery->rec.pointsRead == 0 || pQuery->limit.offset == 0) { + if (pQuery->rec.size == 0 || pQuery->limit.offset == 0) { return; } - if (pQuery->rec.pointsRead <= pQuery->limit.offset) { - pQuery->limit.offset -= pQuery->rec.pointsRead; + if (pQuery->rec.size <= pQuery->limit.offset) { + pQuery->limit.offset -= pQuery->rec.size; - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; // pQuery->pointsOffset = pQuery->rec.pointsToRead; // clear all data in result buffer resetCtxOutputBuf(pRuntimeEnv); @@ -3442,13 +3400,13 @@ void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->status &= (~QUERY_RESBUF_FULL); } else { int32_t numOfSkip = (int32_t)pQuery->limit.offset; - pQuery->rec.pointsRead -= numOfSkip; + pQuery->rec.size -= numOfSkip; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; assert(0); - // memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->pointsRead * bytes); + // memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->size * bytes); pRuntimeEnv->pCtx[i].aOutputBuf += bytes * numOfSkip; if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { @@ -3500,7 +3458,7 @@ static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { SQuery * pQuery = pRuntimeEnv->pQuery; SQueryStatus qStatus = {0}; - if (!needSupplementaryScan(pQuery)) { + if (!needReverseScan(pQuery)) { return; } @@ -3520,7 +3478,7 @@ static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { doScanAllDataBlocks(pRuntimeEnv); queryStatusRestore(pRuntimeEnv, &qStatus); - enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + enableFuncForForwardScan(pRuntimeEnv, pQuery->order.order); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } @@ -3579,7 +3537,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { return toContinue; } -void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { +void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); @@ -3605,10 +3563,7 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { break; } - /* - * set the correct start position, and load the corresponding block in buffer for next - * round scan all data blocks. - */ + // set the correct start position, and load the corresponding block in buffer for next round scan all data blocks. int32_t ret = tsdbDataBlockSeek(pRuntimeEnv->pQueryHandle, pos); status = pQuery->status; @@ -3618,8 +3573,8 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->scanFlag = REPEAT_SCAN; /* check if query is killed or not */ - if (isQueryKilled(pQuery)) { - setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { +// setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } } @@ -3692,18 +3647,13 @@ static bool hasMainOutput(SQuery *pQuery) { return false; } -STableQueryInfo *createMeterQueryInfo(SQInfo *pQInfo, int32_t sid, TSKEY skey, TSKEY ekey) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - +STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, int32_t tid, STimeWindow win) { STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); - pTableQueryInfo->win = (STimeWindow){ - .skey = skey, - .ekey = ekey, - }; - pTableQueryInfo->lastKey = skey; + pTableQueryInfo->win = win; + pTableQueryInfo->lastKey = win.skey; - pTableQueryInfo->sid = sid; + pTableQueryInfo->tid = tid; pTableQueryInfo->cur.vnodeIndex = -1; initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, 100, 100, TSDB_DATA_TYPE_INT); @@ -3719,7 +3669,7 @@ void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) free(pTableQueryInfo); } -void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo, TSKEY skey, TSKEY ekey) { +void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { if (pTableQueryInfo == NULL) { return; } @@ -3923,7 +3873,7 @@ static int32_t getNumOfSubset(SQInfo *pQInfo) { if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (isIntervalQuery(pQuery))) { totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo); } else { -// totalSubset = pQInfo->pSidSet->numOfSubSet; + totalSubset = 1;//pQInfo->pSidSet->numOfSubSet; } return totalSubset; @@ -3959,19 +3909,18 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; int32_t oldOffset = pQInfo->offset; - assert(0); /* * current output space is not enough to keep all the result data of this group, only copy partial results * to SQuery object's result buffer */ - // if (numOfRowsToCopy > pQuery->pointsToRead - numOfResult) { - // numOfRowsToCopy = pQuery->pointsToRead - numOfResult; - // pQInfo->offset += numOfRowsToCopy; - // } else { - // pQInfo->offset = 0; - // pQInfo->subgroupIdx += 1; - // } + if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { + numOfRowsToCopy = pQuery->rec.capacity - numOfResult; + pQInfo->offset += numOfRowsToCopy; + } else { + pQInfo->offset = 0; + pQInfo->subgroupIdx += 1; + } for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; @@ -3982,13 +3931,12 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde } numOfResult += numOfRowsToCopy; - assert(0); - // if (numOfResult == pQuery->rec.pointsToRead) { - // break; - // } + if (numOfResult == pQuery->rec.capacity) { + break; + } } - dTrace("QInfo:%p copy data to SQuery buf completed", GET_QINFO_ADDR(pQuery)); + dTrace("QInfo:%p copy data to SQuery buf completed", pQInfo); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pQuery, numOfResult); @@ -4011,8 +3959,9 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSQL_SO_ASC; int32_t numOfResult = doCopyToSData(pQInfo, result, orderType); - pQuery->rec.pointsRead += numOfResult; - // assert(pQuery->rec.pointsRead <= pQuery->pointsToRead); + pQuery->rec.size += numOfResult; + + assert(pQuery->rec.size <= pQuery->rec.capacity); } static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo) { @@ -4030,9 +3979,8 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInf } } -void stableApplyFunctionsOnBlock_(SQInfo *pQInfo, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, +void stableApplyFunctionsOnBlock(SQueryRuntimeEnv* pRuntimeEnv, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray *pDataBlock, __block_search_fn_t searchFn) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; @@ -4050,36 +3998,6 @@ void stableApplyFunctionsOnBlock_(SQInfo *pQInfo, STableDataInfo *pTableDataInfo updatelastkey(pQuery, pTableQueryInfo); } -// we need to split the refstatsult into different packages. -int32_t vnodeGetResultSize(void *thandle, int32_t *numOfRows) { - SQInfo *pQInfo = (SQInfo *)thandle; - SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; - - /* - * get the file size and set the numOfRows to be the file size, since for tsComp query, - * the returned row size is equalled to 1 - * - * TODO handle the case that the file is too large to send back one time - */ - if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { - struct stat fstat; - if (stat(pQuery->sdata[0]->data, &fstat) == 0) { - *numOfRows = fstat.st_size; - return fstat.st_size; - } else { - dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); - return 0; - } - } else { - return pQuery->rowSize * (*numOfRows); - } -} - -int64_t vnodeGetOffsetVal(void *thandle) { - SQInfo *pQInfo = (SQInfo *)thandle; - return pQInfo->runtimeEnv.pQuery->limit.offset; -} - bool vnodeHasRemainResults(void *handle) { SQInfo *pQInfo = (SQInfo *)handle; @@ -4091,7 +4009,7 @@ bool vnodeHasRemainResults(void *handle) { SQuery * pQuery = pRuntimeEnv->pQuery; SInterpolationInfo *pInterpoInfo = &pRuntimeEnv->interpoInfo; - if (pQuery->limit.limit > 0 && pQInfo->rec.pointsRead >= pQuery->limit.limit) { + if (pQuery->limit.limit > 0 && pQuery->rec.size >= pQuery->limit.limit) { return false; } @@ -4104,7 +4022,7 @@ bool vnodeHasRemainResults(void *handle) { } // query has completed - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision); // int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY @@ -4157,62 +4075,18 @@ static int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, tFilePage **p } static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { -#if 0 - SMeterObj *pObj = pQInfo->pObj; - SQuery * pQuery = &pQInfo->query; - - int tnumOfRows = vnodeList[pObj->vnode].cfg.rowsInFileBlock; - - // for metric query, bufIndex always be 0. - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { // pQInfo->bufIndex == 0 + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { int32_t bytes = pQuery->pSelectExpr[col].resBytes; memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); data += bytes * numOfRows; } -#endif -} - -/** - * Copy the result data/file to output message buffer. - * If the result is in file format, read file from disk and copy to output buffer, compression is not involved since - * data in file is already compressed. - * In case of other result in buffer, compress the result before copy once the tsComressMsg is set. - * - * @param handle - * @param data - * @param numOfRows the number of rows that are not returned in current retrieve - * @return - */ -int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows) { - SQInfo *pQInfo = (SQInfo *)handle; - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - assert(pQuery->pSelectExpr != NULL && pQuery->numOfOutputCols > 0); - - // load data from file to msg buffer - if (isTSCompQuery(pQuery)) { - int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); - - // make sure file exist - if (FD_VALID(fd)) { - size_t s = lseek(fd, 0, SEEK_END); - dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); - - lseek(fd, 0, SEEK_SET); - read(fd, data, s); - close(fd); - - unlink(pQuery->sdata[0]->data); - } else { - dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, - pQuery->sdata[0]->data, strerror(errno)); - } - } else { - doCopyQueryResultToMsg(pQInfo, numOfRows, data); + + // all data returned, set query over + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + setQueryStatus(pQuery, QUERY_OVER); } - - return numOfRows; } int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage **pDataSrc, int32_t numOfRows, @@ -4259,10 +4133,11 @@ int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage } void vnodePrintQueryStatistics(SQInfo *pQInfo) { +#if 0 SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; -#if 0 + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; if (pRuntimeEnv->pResultBuf == NULL) { pSummary->tmpBufferInDisk = 0; @@ -4303,43 +4178,30 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param) { +int32_t doInitializeQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t code = TSDB_CODE_SUCCESS; - // only the successful complete requries the sem_post/over = 1 operations. - if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || - (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { - dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, - pQuery->window.ekey, pQuery->order.order); - - sem_post(&pQInfo->dataReady); - pQInfo->killed = 1; - return TSDB_CODE_SUCCESS; - } - setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQuery, false); - pQInfo->rec = (SResultRec){0}; - // dataInCache requires lastKey value pQuery->lastKey = pQuery->window.skey; - STsdbQueryCond cond = {0}; - - cond.twindow = (STimeWindow) {.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; - cond.order = pQuery->order.order; - cond.colList = *pQuery->colList; + STsdbQueryCond cond = { + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = *pQuery->colList, + }; SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); for (int32_t i = 0; i < pQuery->numOfCols; ++i) { taosArrayPush(cols, &pQuery->colList[i]); } - - pQInfo->runtimeEnv.pQueryHandle = tsdbQueryByTableId(&cond, pQInfo->pTableIdList, cols); - - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; @@ -4350,25 +4212,44 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param) { } // create runtime environment - code = setupQueryRuntimeEnv(&pQInfo->runtimeEnv, NULL, pQuery->order.order, false); + code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { return code; } - pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, false); - if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, isSTableQuery); + + if (isSTableQuery) { int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); if (code != TSDB_CODE_SUCCESS) { return code; } - - int16_t type = TSDB_DATA_TYPE_NULL; - if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); - } else { - type = TSDB_DATA_TYPE_TIMESTAMP; + + if (pQuery->intervalTime == 0) { + int16_t type = TSDB_DATA_TYPE_NULL; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group by columns not tags; + type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + } else { + type = TSDB_DATA_TYPE_INT; // group id + } + + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); + } + + } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + int32_t rows = getInitialPageNum(pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int16_t type = TSDB_DATA_TYPE_NULL; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + } else { + type = TSDB_DATA_TYPE_TIMESTAMP; } initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, rows, 4096, type); @@ -4455,63 +4336,56 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { } } -static void queryOnDataBlocks(SQInfo *pQInfo, STableDataInfo *pMeterDataInfo) { +static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - - // SMeterObj * pTempMeter = getMeterObj(pSupporter->pMetersHashTable, pSupporter->pMeterSidExtInfo[0]->sid); - // __block_search_fn_t searchFn = vnodeSearchKeyFunc[pTempMeter->searchAlgorithm]; - - // dTrace("QInfo:%p start to check data blocks in %d files", pQInfo, pVnodeFileInfo->numOfFiles); - + + int64_t st = taosGetTimestampMs(); + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + tsdb_query_handle_t *pQueryHandle = pRuntimeEnv->pQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { break; } - // prepare the STableDataInfo struct for each table - SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - // SMeterObj * pMeterObj = getMeterObj(pSupporter->pMetersHashTable, blockInfo.sid); - - // pQInfo->pObj = pMeterObj; - // pRuntimeEnv->pMeterObj = pMeterObj; - - STableDataInfo *pTableDataInfo = NULL; - // for (int32_t i = 0; i < pSupporter->pSidSet->numOfTables; ++i) { - // if (pMeterDataInfo[i].pMeterObj == pMeterObj) { - // pTableDataInfo = &pMeterDataInfo[i]; - // break; - // } - // } + STableDataInfo* pTableDataInfo = NULL; + + // todo opt performance + for(int32_t i = 0; i < numOfTables; ++i) { + if (pQInfo->pTableDataInfo[i].pTableQInfo->tid == blockInfo.sid) { + pTableDataInfo = &pQInfo->pTableDataInfo[i]; + break; + } + } - assert(pTableDataInfo != NULL); + assert(pTableDataInfo != NULL && pTableDataInfo->pTableQInfo != NULL); STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; - if (pTableDataInfo->pTableQInfo == NULL) { - // pTableDataInfo->pTableQInfo = createMeterQueryInfo(pQInfo, pMeterObj->sid, pQuery->skey, pQuery->ekey); - } - restoreIntervalQueryRange(pRuntimeEnv, pTableQueryInfo); SDataStatis *pStatis = NULL; SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); TSKEY nextKey = blockInfo.window.ekey; - if (pQuery->intervalTime == 0) { + if (!isIntervalQuery(pQuery)) { setExecutionContext(pQInfo, pTableQueryInfo, pTableDataInfo->tableIndex, pTableDataInfo->groupIdx, nextKey); } else { // interval query setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); int32_t ret = setAdditionalInfo(pQInfo, pTableDataInfo->tableIndex, pTableQueryInfo); + if (ret != TSDB_CODE_SUCCESS) { - // pQInfo->killed = 1; - return; + pQInfo->code = ret; + return taosGetTimestampMs() - st; } } - // stableApplyFunctionsOnBlock_(pSupporter, pTableDataInfo, &blockInfo, pStatis, pDataBlock, searchFn); + stableApplyFunctionsOnBlock(pRuntimeEnv, pTableDataInfo, &blockInfo, pStatis, pDataBlock, binarySearchForKey); } + + int64_t et = taosGetTimestampMs(); + return et - st; } static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *dataInCache, int32_t index, @@ -4595,7 +4469,7 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start pointInterpSupporterSetData(pQInfo, &pointInterpSupporter); pointInterpSupporterDestroy(&pointInterpSupporter); - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); // first/last_row query, do not invoke the finalize for super table query doFinalizeResult(pRuntimeEnv); @@ -4605,7 +4479,7 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start // accumulate the point interpolation result if (numOfRes > 0) { - pQuery->rec.pointsRead += numOfRes; + pQuery->rec.size += numOfRes; forwardCtxOutputBuf(pRuntimeEnv, numOfRes); } @@ -4621,9 +4495,10 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start */ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - -#if 0 SQuery* pQuery = pRuntimeEnv->pQuery; + setQueryStatus(pQuery, QUERY_COMPLETED); + +#if 0 // tSidSet *pTableIdList = pSupporter->pSidSet; int32_t vid = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[0]->sid)->vnode; @@ -4648,7 +4523,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pSupporter->meterIdx = start; for (int32_t k = start; k <= end; ++k, pSupporter->meterIdx++) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -4675,7 +4550,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pSupporter->subgroupIdx); for (int32_t k = start; k <= end; ++k) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -4693,7 +4568,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pSupporter->subgroupIdx++; // output buffer is full, return to client - if (pQuery->pointsRead >= pQuery->pointsToRead) { + if (pQuery->size >= pQuery->pointsToRead) { break; } } @@ -4709,9 +4584,9 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { */ if (pSupporter->subgroupIdx > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); - pQInfo->pointsRead += pQuery->pointsRead; + pQInfo->size += pQuery->size; - if (pQuery->pointsRead > 0) { + if (pQuery->size > 0) { return; } } @@ -4726,7 +4601,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { while (pSupporter->meterIdx < pSupporter->numOfMeters) { int32_t k = pSupporter->meterIdx; - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -4775,9 +4650,9 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { } } - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); - pQuery->pointsRead = getNumOfResult(pRuntimeEnv); + pQuery->size = getNumOfResult(pRuntimeEnv); doSkipResults(pRuntimeEnv); // the limitation of output result is reached, set the query completed @@ -4812,7 +4687,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pQuery->skey = pQuery->lastKey; // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter - if (pQuery->pointsRead == 0) { + if (pQuery->size == 0) { assert(!Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); continue; } else { @@ -4859,110 +4734,100 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { } pQInfo->pTableQuerySupporter->subgroupIdx = 0; - pQuery->pointsRead = 0; + pQuery->size = 0; copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult); } - pQInfo->pointsRead += pQuery->pointsRead; + pQInfo->size += pQuery->size; pQuery->pointsOffset = pQuery->pointsToRead; dTrace( "QInfo %p vid:%d, numOfMeters:%d, index:%d, numOfGroups:%d, %d points returned, totalRead:%d totalReturn:%d," "next skey:%" PRId64 ", offset:%" PRId64, - pQInfo, vid, pTableIdList->numOfTables, pSupporter->meterIdx, pTableIdList->numOfSubSet, pQuery->pointsRead, pQInfo->pointsRead, + pQInfo, vid, pTableIdList->numOfTables, pSupporter->meterIdx, pTableIdList->numOfSubSet, pQuery->size, pQInfo->size, pQInfo->pointsReturned, pQuery->skey, pQuery->limit.offset); #endif } -static void doOrderedScan(SQInfo *pQInfo) { - SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; -#if 0 -// if (pQInfo->runtimeEnv. == NULL) { -// pSupporter->pMeterDataInfo = calloc(pSupporter->pSidSet->numOfTables, sizeof(STableDataInfo)); -// } - - STableIdInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo; +static void createTableDataInfo(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - tSidSet* pSidset = pSupporter->pSidSet; - int32_t groupId = 0; + // todo make sure the table are added the reference count to gauranteed that all involved tables are valid + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); - for (int32_t i = 0; i < pSidset->numOfTables; ++i) { // load all meter meta info - SMeterObj *pMeterObj = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[i]->sid); - if (pMeterObj == NULL) { - dError("QInfo:%p failed to find required sid:%d", pQInfo, pMeterSidExtInfo[i]->sid); - continue; + if (pQInfo->pTableDataInfo == NULL) { + pQInfo->pTableDataInfo = (STableDataInfo *)calloc(1, sizeof(STableDataInfo) * numOfTables); + if (pQInfo->pTableDataInfo == NULL) { + dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); + pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; + return; } - if (i >= pSidset->starterPos[groupId + 1]) { - groupId += 1; + int32_t groupId = 0; + for (int32_t i = 0; i < numOfTables; ++i) { // load all meter meta info + STableId *id = taosArrayGet(pQInfo->pTableIdList, i); + STableDataInfo *pInfo = &pQInfo->pTableDataInfo[i]; + + setTableDataInfo(pInfo, i, groupId); + pInfo->pTableQInfo = createTableQueryInfo(&pQInfo->runtimeEnv, id->tid, pQuery->window); } - - STableDataInfo *pOneMeterDataInfo = &pSupporter->pMeterDataInfo[i]; - assert(pOneMeterDataInfo->pMeterObj == NULL); - - setMeterDataInfo(pOneMeterDataInfo, pMeterObj, i, groupId); - pOneMeterDataInfo->pTableQInfo = createMeterQueryInfo(pSupporter, pMeterObj->sid, pQuery->skey, pQuery->ekey); } - - queryOnDataBlocks(pQInfo, pSupporter->pMeterDataInfo); - if (pQInfo->code != TSDB_CODE_SUCCESS) { - return; - } -#endif } -static void setupMeterQueryInfoForSupplementQuery(SQInfo *pQInfo) { +static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - int32_t num = taosHashGetSize(pQInfo->pTableIdList); - for (int32_t i = 0; i < num; ++i) { - // STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; - // changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo, pSupporter->rawSKey, pSupporter->rawEKey); + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + + for (int32_t i = 0; i < numOfTables; ++i) { + STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); } } -static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - - if (!needSupplementaryScan(pQuery)) { - dTrace("QInfo:%p no need to do supplementary scan, query completed", pQInfo); - return; - } - +static void doSaveContext(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); - // disableFunctForSuppleScan(pSupporter, pQuery->order.order); - + disableFuncForReverseScan(pQInfo, pQuery->order.order); + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1u; } - -#if 0 - SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); - setupMeterQueryInfoForSupplementQuery(pSupporter); - int64_t st = taosGetTimestampMs(); - - doOrderedScan(pQInfo); - - int64_t et = taosGetTimestampMs(); - dTrace("QInfo:%p supplementary scan completed, elapsed time: %lldms", pQInfo, et - st); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + prepareQueryInfoForReverseScan(pQInfo); +} + +static void doRestoreContext(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; - /* - * restore the env - * the meter query info is not reset to the original state - */ - SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); - enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; } -#endif + + enableFuncForForwardScan(pRuntimeEnv, pQuery->order.order); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } -static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { +static void doCloseAllTimeWindowAfterScan(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + + if (isIntervalQuery(pQuery)) { + for (int32_t i = 0; i < numOfTables; ++i) { + STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + closeAllTimeWindow(&pTableQueryInfo->windowResInfo); + } + } else { // close results for group result + closeAllTimeWindow(&pQInfo->runtimeEnv.windowResInfo); + } +} + +static void multiTableQueryProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -4971,7 +4836,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { * if the subgroupIdx > 0, the query process must be completed yet, we only need to * copy the data into output buffer */ - if (pQuery->intervalTime > 0) { + if (isIntervalQuery(pQuery)) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW @@ -4981,56 +4846,59 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); } - pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + pQuery->rec.size += pQuery->rec.size; - if (pQuery->rec.pointsRead == 0) { + if (pQuery->rec.size == 0) { // vnodePrintQueryStatistics(pSupporter); } - dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->rec.pointsRead, - pQInfo->rec.pointsRead, pQInfo->pointsReturned); - return; - } -#if 0 - pSupporter->pMeterDataInfo = (STableDataInfo *)calloc(1, sizeof(STableDataInfo) * pSupporter->numOfMeters); - if (pSupporter->pMeterDataInfo == NULL) { - dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); - pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; + dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.size, pQuery->rec.total); return; } - dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, group:%d", pQInfo, pSupporter->rawSKey, - pSupporter->rawEKey, pQuery->order.order, pSupporter->pSidSet->numOfSubSet); + dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, pQuery->window.skey, + pQuery->window.ekey, pQuery->order.order); - dTrace("QInfo:%p main query scan start", pQInfo); - int64_t st = taosGetTimestampMs(); - doOrderedScan(pQInfo); + // create the query support structures + createTableDataInfo(pQInfo); - int64_t et = taosGetTimestampMs(); - dTrace("QInfo:%p main scan completed, elapsed time: %lldms, supplementary scan start, order:%d", pQInfo, et - st, + // do check all qualified data blocks + int64_t el = queryOnDataBlocks(pQInfo); + dTrace("QInfo:%p forward scan completed, elapsed time: %lldms, reversed scan start, order:%d", pQInfo, el, pQuery->order.order ^ 1u); - if (pQuery->intervalTime > 0) { - for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) { - STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; - closeAllTimeWindow(&pTableQueryInfo->windowResInfo); - } - } else { // close results for group result - closeAllTimeWindow(&pRuntimeEnv->windowResInfo); + // query error occurred or query is killed, abort current execution + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { + dTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); + return; + } + + // close all time window results + doCloseAllTimeWindowAfterScan(pQInfo); + + if (needReverseScan(pQuery)) { + doSaveContext(pQInfo); + + el = queryOnDataBlocks(pQInfo); + dTrace("QInfo:%p reversed scan completed, elapsed time: %lldms", pQInfo, el); + + doRestoreContext(pQInfo); + } else { + dTrace("QInfo:%p no need to do reversed scan, query completed", pQInfo); } - doMultiMeterSupplementaryScan(pQInfo); + setQueryStatus(pQuery, QUERY_COMPLETED); - if (isQueryKilled(pQuery)) { - dTrace("QInfo:%p query killed, abort", pQInfo); + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { + dTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; } - if (pQuery->intervalTime > 0 || isSumAvgRateQuery(pQuery)) { - assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); + if (isIntervalQuery(pQuery) || isSumAvgRateQuery(pQuery)) { +// assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); - if (mergeMetersResultToOneGroups(pSupporter) == TSDB_CODE_SUCCESS) { - copyResToQueryResultBuf(pSupporter, pQuery); + if (mergeResultsToGroup(pQInfo) == TSDB_CODE_SUCCESS) { + copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); @@ -5041,10 +4909,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { } // handle the limitation of output buffer - pQInfo->pointsRead += pQuery->pointsRead; - dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, - pQInfo->pointsReturned); -#endif + dTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); } /* @@ -5057,17 +4922,17 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } // since the numOfOutputElems must be identical for all sql functions that are allowed to be executed simutanelously. - pQuery->rec.pointsRead = getNumOfResult(pRuntimeEnv); - // assert(pQuery->pointsRead <= pQuery->pointsToRead && - // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)); + pQuery->rec.size = getNumOfResult(pRuntimeEnv); + // assert(pQuery->size <= pQuery->pointsToRead && + // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED)); // must be top/bottom query if offset > 0 if (pQuery->limit.offset > 0) { @@ -5077,15 +4942,12 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { doSkipResults(pRuntimeEnv); doRevisedResultsByLimit(pQInfo); - pQInfo->rec.pointsRead = pQuery->rec.pointsRead; + pQuery->rec.size = pQuery->rec.size; } static void tableMultiOutputProcessor(SQInfo *pQInfo) { -#if 0 - SQuery * pQuery = &pQInfo->query; - SMeterObj *pMeterObj = pQInfo->pObj; - - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pTableQuerySupporter->runtimeEnv; + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { @@ -5093,66 +4955,55 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { } while (1) { - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } - pQuery->pointsRead = getNumOfResult(pRuntimeEnv); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->pointsRead > 0) { + pQuery->rec.size = getNumOfResult(pRuntimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.size > 0) { doSkipResults(pRuntimeEnv); } /* - * 1. if pQuery->pointsRead == 0, pQuery->limit.offset >= 0, still need to check data - * 2. if pQuery->pointsRead > 0, pQuery->limit.offset must be 0 + * 1. if pQuery->size == 0, pQuery->limit.offset >= 0, still need to check data + * 2. if pQuery->size > 0, pQuery->limit.offset must be 0 */ - if (pQuery->pointsRead > 0 || Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + if (pQuery->rec.size > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { break; } - TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); - assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK))); - dTrace("QInfo:%p vid:%d sid:%d id:%s, skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64, - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->limit.offset, pQuery->lastKey, - pQuery->ekey); + pQInfo, pQuery->limit.offset, pQuery->lastKey); resetCtxOutputBuf(pRuntimeEnv); } doRevisedResultsByLimit(pQInfo); - pQInfo->pointsRead += pQuery->pointsRead; - - if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)) { - TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); - assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK))); - - dTrace("QInfo:%p vid:%d sid:%d id:%s, query abort due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->lastKey, pQuery->ekey); + if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { + dTrace("QInfo:%p query paused due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, + pQInfo, pQuery->lastKey, pQuery->window.ekey); } - dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, - pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); +// dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, +// pMeterObj->sid, pMeterObj->meterId, pQuery->size, pQInfo->size, pQInfo->pointsReturned); - pQuery->pointsOffset = pQuery->pointsToRead; // restore the available buffer - if (!isTSCompQuery(pQuery)) { - assert(pQuery->pointsRead <= pQuery->pointsToRead); - } - -#endif +// pQuery->pointsOffset = pQuery->pointsToRead; //restore the available buffer +// if (!isTSCompQuery(pQuery)) { +// assert(pQuery->size <= pQuery->pointsToRead); +// } } -static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { +static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; while (1) { initCtxOutputBuf(pRuntimeEnv); - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return; } @@ -5171,34 +5022,25 @@ static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= c; } - if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { - break; - } - - // load the data block for the next retrieve - // loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED|QUERY_RESBUF_FULL)) { break; } } } -/* handle time interval query on single table */ +// handle time interval query on table static void tableIntervalProcessor(SQInfo *pQInfo) { - // STable *pMeterObj = pQInfo->pObj; - SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); SQuery * pQuery = pRuntimeEnv->pQuery; int32_t numOfInterpo = 0; while (1) { - resetCtxOutputBuf(pRuntimeEnv); - vnodeSingleMeterIntervalMainLooper(pRuntimeEnv); + tableIntervalProcessImpl(pRuntimeEnv); - if (pQuery->intervalTime > 0) { + if (isIntervalQuery(pQuery)) { pQInfo->subgroupIdx = 0; // always start from 0 - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); @@ -5209,57 +5051,48 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { doRevisedResultsByLimit(pQInfo); break; } else { - taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.pointsRead, pQuery->interpoType); + taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.size, pQuery->interpoType); SData **pInterpoBuf = pRuntimeEnv->pInterpoBuf; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.pointsRead * pQuery->pSelectExpr[i].resBytes); + memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.size * pQuery->pSelectExpr[i].resBytes); } numOfInterpo = 0; - pQuery->rec.pointsRead = vnodeQueryResultInterpolate( - pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.pointsRead, &numOfInterpo); + pQuery->rec.size = vnodeQueryResultInterpolate( + pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.size, &numOfInterpo); - dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.pointsRead); - if (pQuery->rec.pointsRead > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.size); + if (pQuery->rec.size > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { doRevisedResultsByLimit(pQInfo); break; } // no result generated yet, continue retrieve data - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; } } // all data scanned, the group by normal column can return if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // todo refactor with merge interval time result pQInfo->subgroupIdx = 0; - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); } - pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + pQuery->rec.size += pQuery->rec.size; pQInfo->pointsInterpo += numOfInterpo; // dTrace("%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d // totalReturn:%d", - // pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, numOfInterpo, - // pQInfo->pointsRead - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); + // pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->size, numOfInterpo, + // pQInfo->size - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); } -void qTableQuery(SQInfo* pQInfo) { - assert(pQInfo != NULL); - - if (pQInfo->killed) { - dTrace("QInfo:%p it is already killed, abort", pQInfo); - return; - } - - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; - -// dTrace("vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d, QInfo:%p", pQInfo); +static void singleTableQueryImpl(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; if (vnodeHasRemainResults(pQInfo)) { /* @@ -5267,64 +5100,56 @@ void qTableQuery(SQInfo* pQInfo) { * So, we do keep in this procedure instead of launching retrieve procedure for next results. */ int32_t numOfInterpo = 0; - int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); - pQuery->rec.pointsRead = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, - (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); + pQuery->rec.size = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, + (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); doRevisedResultsByLimit(pQInfo); pQInfo->pointsInterpo += numOfInterpo; - pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + pQuery->rec.size += pQuery->rec.size; -// dTrace("QInfo:%p %d points returned %d points interpo, totalRead:%d totalInterpo:%d totalReturn:%d", -// pQInfo, pQuery->pointsRead, numOfInterpo, pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); + dTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } // here we have scan all qualified data in both data file and cache - if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { // continue to get push data from the group result if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || - (pQuery->intervalTime > 0 && pQInfo->pointsReturned < pQuery->limit.limit)) { + ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { + // todo limit the output for interval query? - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; pQInfo->subgroupIdx = 0; // always start from 0 if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); - pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + pQuery->rec.size += pQuery->rec.size; clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); - if (pQuery->rec.pointsRead > 0) { -// dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d totalReturn:%d", -// pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, -// pQInfo->pointsInterpo, pQInfo->pointsReturned); - + if (pQuery->rec.size > 0) { + dTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } } } -// dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, pMeterObj->sid, -// pMeterObj->meterId, pQInfo->pointsRead); - -// vnodePrintQueryStatistics(pSupporter); + dTrace("QInfo:%p query over, %d rows are returned", pQInfo, pQuery->rec.total); + // vnodePrintQueryStatistics(pSupporter); sem_post(&pQInfo->dataReady); return; } // number of points returned during this query - pQuery->rec.pointsRead = 0; - + pQuery->rec.size = 0; int64_t st = taosGetTimestampUs(); // group by normal column, sliding window query, interval query are handled by interval query processor if (pQuery->intervalTime != 0 || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) -// assert(pQuery->checkBufferInLoop == 0 && pQuery->pointsOffset == pQuery->pointsToRead); tableIntervalProcessor(pQInfo); } else { if (isFixedOutputQuery(pQuery)) { @@ -5341,41 +5166,24 @@ void qTableQuery(SQInfo* pQInfo) { pQInfo->elapsedTime += (taosGetTimestampUs() - st); /* check if query is killed or not */ - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed", pQInfo); -// pQInfo->over = 1; } else { -// dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", pQInfo, -// pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); + dTrace("QInfo:%p query task completed, %d points are returned", pQInfo, pQuery->rec.size); } sem_post(&pQInfo->dataReady); -// vnodeDecRefCount(pQInfo); } -void qSuperTableQuery(void *pReadMsg) { -// SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; -// -// if (pQInfo == NULL) { -// return; -// } - -// if (pQInfo->killed) { -// vnodeDecRefCount(pQInfo); -// dTrace("QInfo:%p it is already killed, abort", pQInfo); -// return; -// } - -// assert(pQInfo->refCount >= 1); -#if 0 - SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; - pQuery->rec.pointsRead = 0; +static void multiTableQueryImpl(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + pQuery->rec.size = 0; int64_t st = taosGetTimestampUs(); - if (pQuery->intervalTime > 0 || + + if (isIntervalQuery(pQuery) || (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { - assert(pQuery->checkBufferInLoop == 0); - vnodeMultiMeterQueryProcessor(pQInfo); + multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBufferInLoop == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)); @@ -5383,30 +5191,24 @@ void qSuperTableQuery(void *pReadMsg) { vnodeSTableSeqProcessor(pQInfo); } - /* record the total elapsed time */ + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); - pQuery->status = isQueryKilled(pQuery) ? 1 : 0; +// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); -// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->pointsRead, -// pQInfo->query.interpoType); - - if (pQuery->rec.pointsRead == 0) { -// pQInfo->over = 1; -// dTrace("QInfo:%p over, %d meters queried, %d points are returned", pQInfo, pSupporter->numOfMeters, -// pQInfo->pointsRead); + if (pQuery->rec.size == 0) { + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + dTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, numOfTables, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); } sem_post(&pQInfo->dataReady); -// vnodeDecRefCount(pQInfo); -#endif } -static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryTableMsg, SSqlFuncExprMsg *pExprMsg) { +static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { int32_t j = 0; - while (j < pQueryTableMsg->numOfCols) { - if (pExprMsg->colInfo.colId == pQueryTableMsg->colList[j].colId) { + while (j < pQueryMsg->numOfCols) { + if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { break; } @@ -5416,104 +5218,127 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryTableMsg, SSqlFuncEx return j; } -bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryTableMsg, SSqlFuncExprMsg *pExprMsg) { - int32_t j = getColumnIndexInSource(pQueryTableMsg, pExprMsg); - return j < pQueryTableMsg->numOfCols; +bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { + int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg); + return j < pQueryMsg->numOfCols; } -static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryTableMsg) { - if (pQueryTableMsg->intervalTime < 0) { - dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryTableMsg, pQueryTableMsg->intervalTime); - return -1; - } - - if (pQueryTableMsg->numOfTagsCols < 0 || pQueryTableMsg->numOfTagsCols > TSDB_MAX_TAGS + 1) { - dError("qmsg:%p illegal value of numOfTagsCols %d", pQueryTableMsg, pQueryTableMsg->numOfTagsCols); - return -1; - } - - if (pQueryTableMsg->numOfCols <= 0 || pQueryTableMsg->numOfCols > TSDB_MAX_COLUMNS) { - dError("qmsg:%p illegal value of numOfCols %d", pQueryTableMsg, pQueryTableMsg->numOfCols); +static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { + if (pQueryMsg->intervalTime < 0) { + dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime); return -1; } - if (pQueryTableMsg->numOfTables <= 0) { - dError("qmsg:%p illegal value of numOfTables %d", pQueryTableMsg, pQueryTableMsg->numOfTables); + if (pQueryMsg->numOfCols <= 0 || pQueryMsg->numOfCols > TSDB_MAX_COLUMNS) { + dError("qmsg:%p illegal value of numOfCols %d", pQueryMsg, pQueryMsg->numOfCols); return -1; } - if (pQueryTableMsg->numOfGroupCols < 0) { - dError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryTableMsg, pQueryTableMsg->numOfGroupCols); + if (pQueryMsg->numOfTables <= 0) { + dError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables); return -1; } - if (pQueryTableMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryTableMsg->numOfOutputCols <= 0) { - dError("qmsg:%p illegal value of output columns %d", pQueryTableMsg, pQueryTableMsg->numOfOutputCols); + if (pQueryMsg->numOfGroupCols < 0) { + dError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryMsg, pQueryMsg->numOfGroupCols); return -1; } - if (pQueryTableMsg->tagLength < 0) { - dError("qmsg:%p illegal value of tag length %d", pQueryTableMsg, pQueryTableMsg->tagLength); + if (pQueryMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutputCols <= 0) { + dError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutputCols); return -1; } return 0; } -static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray** pTableIdList) { - pQueryTableMsg->vgId = htons(pQueryTableMsg->vgId); - pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); +static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** pTableIdList) { + assert(pQueryMsg->numOfTables > 0); - pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); - pQueryTableMsg->window.ekey = htobe64(pQueryTableMsg->window.ekey); - - pQueryTableMsg->order = htons(pQueryTableMsg->order); - pQueryTableMsg->orderColId = htons(pQueryTableMsg->orderColId); - - pQueryTableMsg->queryType = htons(pQueryTableMsg->queryType); - - pQueryTableMsg->intervalTime = htobe64(pQueryTableMsg->intervalTime); - pQueryTableMsg->slidingTime = htobe64(pQueryTableMsg->slidingTime); - - pQueryTableMsg->numOfTagsCols = htons(pQueryTableMsg->numOfTagsCols); - pQueryTableMsg->numOfCols = htons(pQueryTableMsg->numOfCols); - pQueryTableMsg->numOfOutputCols = htons(pQueryTableMsg->numOfOutputCols); - pQueryTableMsg->numOfGroupCols = htons(pQueryTableMsg->numOfGroupCols); - pQueryTableMsg->tagLength = htons(pQueryTableMsg->tagLength); - - pQueryTableMsg->limit = htobe64(pQueryTableMsg->limit); - pQueryTableMsg->offset = htobe64(pQueryTableMsg->offset); + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableId)); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->sid = htonl(pTableIdInfo->sid); + pTableIdInfo->uid = htobe64(pTableIdInfo->uid); + pTableIdInfo->key = htobe64(pTableIdInfo->key); + + STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->sid}; + taosArrayPush(*pTableIdList, &id); - pQueryTableMsg->tsOffset = htonl(pQueryTableMsg->tsOffset); - pQueryTableMsg->tsLen = htonl(pQueryTableMsg->tsLen); - pQueryTableMsg->tsNumOfBlocks = htonl(pQueryTableMsg->tsNumOfBlocks); - pQueryTableMsg->tsOrder = htonl(pQueryTableMsg->tsOrder); + pMsg += sizeof(STableIdInfo); + + for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) { + pTableIdInfo = (STableIdInfo *)pMsg; + + pTableIdInfo->sid = htonl(pTableIdInfo->sid); + pTableIdInfo->uid = htobe64(pTableIdInfo->uid); + pTableIdInfo->key = htobe64(pTableIdInfo->key); + + taosArrayPush(*pTableIdList, pTableIdInfo); + pMsg += sizeof(STableIdInfo); + } + return pMsg; +} + +/** + * pQueryMsg->head has been converted before this function is called. + * + * @param pQueryMsg + * @param pTableIdList + * @param pExpr + * @return + */ +static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, + wchar_t** tagCond) { + pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); + + pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); + pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime); + pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); + pQueryMsg->limit = htobe64(pQueryMsg->limit); + pQueryMsg->offset = htobe64(pQueryMsg->offset); + + pQueryMsg->order = htons(pQueryMsg->order); + pQueryMsg->orderColId = htons(pQueryMsg->orderColId); + pQueryMsg->queryType = htons(pQueryMsg->queryType); + + pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); + pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols); + pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); + pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->nameCondLen = htons(pQueryMsg->nameCondLen); + pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); + pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); + pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); + pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); + // query msg safety check - if (validateQueryMeterMsg(pQueryTableMsg) != 0) { + if (validateQueryMsg(pQueryMsg) != 0) { return TSDB_CODE_INVALID_QUERY_MSG; } - char *pMsg = (char *)(pQueryTableMsg->colList) + sizeof(SColumnInfo) * pQueryTableMsg->numOfCols; - - for (int32_t col = 0; col < pQueryTableMsg->numOfCols; ++col) { - pQueryTableMsg->colList[col].colId = htons(pQueryTableMsg->colList[col].colId); - pQueryTableMsg->colList[col].type = htons(pQueryTableMsg->colList[col].type); - pQueryTableMsg->colList[col].bytes = htons(pQueryTableMsg->colList[col].bytes); - pQueryTableMsg->colList[col].numOfFilters = htons(pQueryTableMsg->colList[col].numOfFilters); + char *pMsg = (char *)(pQueryMsg->colList) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; - assert(pQueryTableMsg->colList[col].type >= TSDB_DATA_TYPE_BOOL && - pQueryTableMsg->colList[col].type <= TSDB_DATA_TYPE_NCHAR); + for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { + SColumnInfo* pColInfo = &pQueryMsg->colList[col]; + + pColInfo->colId = htons(pColInfo->colId); + pColInfo->type = htons(pColInfo->type); + pColInfo->bytes = htons(pColInfo->bytes); + pColInfo->numOfFilters = htons(pColInfo->numOfFilters); - int32_t numOfFilters = pQueryTableMsg->colList[col].numOfFilters; + assert(pColInfo->type >= TSDB_DATA_TYPE_BOOL && pColInfo->type <= TSDB_DATA_TYPE_NCHAR); + int32_t numOfFilters = pColInfo->numOfFilters; if (numOfFilters > 0) { - pQueryTableMsg->colList[col].filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); + pColInfo->filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); } for (int32_t f = 0; f < numOfFilters; ++f) { SColumnFilterInfo *pFilterInfo = (SColumnFilterInfo *)pMsg; - SColumnFilterInfo *pDestFilterInfo = &pQueryTableMsg->colList[col].filters[f]; + SColumnFilterInfo *pDestFilterInfo = &pColInfo->filters[f]; pDestFilterInfo->filterOnBinary = htons(pFilterInfo->filterOnBinary); @@ -5537,27 +5362,22 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray** pTa bool hasArithmeticFunction = false; - /* - * 1. simple projection query on meters, we only record the pSqlFuncExprs[i].colIdx value - * 2. for complex queries, whole SqlExprs object is required. - */ - pQueryTableMsg->pSqlFuncExprs = (int64_t)malloc(POINTER_BYTES * pQueryTableMsg->numOfOutputCols); + *pExpr = calloc(pQueryMsg->numOfOutputCols, POINTER_BYTES); SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; - for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { - ((SSqlFuncExprMsg **)pQueryTableMsg->pSqlFuncExprs)[i] = pExprMsg; + for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + (*pExpr)[i] = pExprMsg; pExprMsg->colInfo.colIdx = htons(pExprMsg->colInfo.colIdx); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); pMsg += sizeof(SSqlFuncExprMsg); for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); + pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); pExprMsg->arg[j].argBytes = htons(pExprMsg->arg[j].argBytes); if (pExprMsg->arg[j].argType == TSDB_DATA_TYPE_BINARY) { @@ -5576,7 +5396,7 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray** pTa return TSDB_CODE_INVALID_QUERY_MSG; } } else { - if (!vnodeValidateExprColumnInfo(pQueryTableMsg, pExprMsg)) { + if (!vnodeValidateExprColumnInfo(pQueryMsg, pExprMsg)) { return TSDB_CODE_INVALID_QUERY_MSG; } } @@ -5584,74 +5404,55 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray** pTa pExprMsg = (SSqlFuncExprMsg *)pMsg; } - pQueryTableMsg->colNameLen = htonl(pQueryTableMsg->colNameLen); + pQueryMsg->colNameLen = htonl(pQueryMsg->colNameLen); if (hasArithmeticFunction) { // column name array - assert(pQueryTableMsg->colNameLen > 0); - pQueryTableMsg->colNameList = (int64_t)pMsg; - pMsg += pQueryTableMsg->colNameLen; + assert(pQueryMsg->colNameLen > 0); + pQueryMsg->colNameList = (int64_t)pMsg; + pMsg += pQueryMsg->colNameLen; } - *pTableIdList = taosArrayInit(pQueryTableMsg->numOfTables, sizeof(STableIdInfo)); - - STableIdInfo* pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->sid = htonl(pTableIdInfo->sid); - pTableIdInfo->uid = htobe64(pTableIdInfo->uid); - pTableIdInfo->key = htobe64(pTableIdInfo->key); - - taosArrayPush(*pTableIdList, pTableIdInfo); - pMsg += sizeof(STableIdInfo); - - for (int32_t j = 1; j < pQueryTableMsg->numOfTables; ++j) { - pTableIdInfo = (STableIdInfo *)pMsg; - - pTableIdInfo->sid = htonl(pTableIdInfo->sid); - pTableIdInfo->uid = htobe64(pTableIdInfo->uid); - pTableIdInfo->key = htobe64(pTableIdInfo->key); - - taosArrayPush(*pTableIdList, pTableIdInfo); - pMsg += sizeof(STableIdInfo); - } - - if (pQueryTableMsg->numOfGroupCols > 0 || pQueryTableMsg->numOfTagsCols > 0) { // group by tag columns - pQueryTableMsg->pTagSchema = (uint64_t)pMsg; - SSchema *pTagSchema = (SSchema *)pQueryTableMsg->pTagSchema; - pMsg += sizeof(SSchema) * pQueryTableMsg->numOfTagsCols; + pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList); - if (pQueryTableMsg->numOfGroupCols > 0) { - pQueryTableMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryTableMsg->numOfTagsCols]); - } else { - pQueryTableMsg->groupbyTagIds = 0; - } - pQueryTableMsg->orderByIdx = htons(pQueryTableMsg->orderByIdx); - pQueryTableMsg->orderType = htons(pQueryTableMsg->orderType); + if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns +// if (pQueryMsg->numOfGroupCols > 0) { +// pQueryMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryMsg->numOfTagsCols]); +// } else { +// pQueryMsg->groupbyTagIds = 0; +// } + pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); + pQueryMsg->orderType = htons(pQueryMsg->orderType); - pMsg += sizeof(SColIndexEx) * pQueryTableMsg->numOfGroupCols; + pMsg += sizeof(SColIndexEx) * pQueryMsg->numOfGroupCols; } else { - pQueryTableMsg->pTagSchema = 0; - pQueryTableMsg->groupbyTagIds = 0; + pQueryMsg->groupbyTagIds = 0; } - pQueryTableMsg->interpoType = htons(pQueryTableMsg->interpoType); - if (pQueryTableMsg->interpoType != TSDB_INTERPO_NONE) { - pQueryTableMsg->defaultVal = (uint64_t)(pMsg); + pQueryMsg->interpoType = htons(pQueryMsg->interpoType); + if (pQueryMsg->interpoType != TSDB_INTERPO_NONE) { + pQueryMsg->defaultVal = (uint64_t)(pMsg); int64_t *v = (int64_t *)pMsg; - for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { v[i] = htobe64(v[i]); } + + pMsg += sizeof(int64_t) * pQueryMsg->numOfOutputCols; + } + + // the tag query condition expression string is located at the end of query msg + if (pQueryMsg->tagCondLen > 0) { + *tagCond = calloc(1, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); + memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); } - dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 - ", numOfGroupbyTagCols:%d, numOfTagCols:%d, timestamp order:%d, " - "tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 - ", fillType:%d, comptslen:%d, limit:%" PRId64 - ", " - "offset:%" PRId64, - pQueryTableMsg, pQueryTableMsg->numOfTables, pQueryTableMsg->window.skey, pQueryTableMsg->window.ekey, - pQueryTableMsg->numOfGroupCols, pQueryTableMsg->numOfTagsCols, pQueryTableMsg->order, - pQueryTableMsg->orderType, pQueryTableMsg->orderByIdx, pQueryTableMsg->numOfOutputCols, - pQueryTableMsg->numOfCols, pQueryTableMsg->intervalTime, pQueryTableMsg->interpoType, pQueryTableMsg->tsLen, - pQueryTableMsg->limit, pQueryTableMsg->offset); + dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, " + "timestamp order:%d, tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 + ", fillType:%d, comptslen:%d, limit:%" PRId64 ", offset:%" PRId64, + pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, + pQueryMsg->numOfGroupCols, pQueryMsg->order, pQueryMsg->orderType, + pQueryMsg->orderByIdx, pQueryMsg->numOfOutputCols, + pQueryMsg->numOfCols, pQueryMsg->intervalTime, pQueryMsg->interpoType, pQueryMsg->tsLen, + pQueryMsg->limit, pQueryMsg->offset); return 0; } @@ -5707,22 +5508,20 @@ static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMs return TSDB_CODE_SUCCESS; } -static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr) { +static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr, SSqlFuncExprMsg** pExprMsg) { *pSqlFuncExpr = NULL; int32_t code = TSDB_CODE_SUCCESS; - + SSqlFunctionExpr *pExprs = (SSqlFunctionExpr *)calloc(1, sizeof(SSqlFunctionExpr) * pQueryMsg->numOfOutputCols); if (pExprs == NULL) { - tfree(pQueryMsg->pSqlFuncExprs); return TSDB_CODE_SERV_OUT_OF_MEMORY; } bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); int16_t tagLen = 0; - SSchema *pTagSchema = (SSchema *)pQueryMsg->pTagSchema; for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { - pExprs[i].pBase = *((SSqlFuncExprMsg **)pQueryMsg->pSqlFuncExprs)[i]; + pExprs[i].pBase = *pExprMsg[i]; pExprs[i].resBytes = 0; int16_t type = 0; @@ -5730,36 +5529,24 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct SColIndexEx *pColumnIndexExInfo = &pExprs[i].pBase.colInfo; - // tag column schema is kept in pQueryMsg->pColumnModel - if (TSDB_COL_IS_TAG(pColumnIndexExInfo->flag)) { - if (pColumnIndexExInfo->colIdx >= pQueryMsg->numOfTagsCols) { - tfree(pExprs); + // parse the arithmetic expression + if (pExprs[i].pBase.functionId == TSDB_FUNC_ARITHM) { + code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); - return TSDB_CODE_INVALID_QUERY_MSG; + if (code != TSDB_CODE_SUCCESS) { + tfree(pExprs); + return code; } - type = pTagSchema[pColumnIndexExInfo->colIdx].type; - bytes = pTagSchema[pColumnIndexExInfo->colIdx].bytes; - - } else { // parse the arithmetic expression - if (pExprs[i].pBase.functionId == TSDB_FUNC_ARITHM) { - code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); - - if (code != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return code; - } - - type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypeDesc[type].nSize; - } else { // parse the normal column - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); - assert(j < pQueryMsg->numOfCols); + type = TSDB_DATA_TYPE_DOUBLE; + bytes = tDataTypeDesc[type].nSize; + } else { // parse the normal column + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); + assert(j < pQueryMsg->numOfCols); - SColumnInfo* pCol = &pQueryMsg->colList[j]; - type = pCol->type; - bytes = pCol->bytes; - } + SColumnInfo *pCol = &pQueryMsg->colList[j]; + type = pCol->type; + bytes = pCol->bytes; } int32_t param = pExprs[i].pBase.arg[0].argValue.i64; @@ -5779,7 +5566,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct // TODO refactor for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { - pExprs[i].pBase = *((SSqlFuncExprMsg **)pQueryMsg->pSqlFuncExprs)[i]; + pExprs[i].pBase = *pExprMsg[i]; int16_t functId = pExprs[i].pBase.functionId; if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); @@ -5796,7 +5583,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct } } - tfree(pQueryMsg->pSqlFuncExprs); + tfree(pExprMsg); *pSqlFuncExpr = pExprs; return TSDB_CODE_SUCCESS; @@ -5909,7 +5696,41 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_SUCCESS; } -static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pExprs, SArray* pTableIdList) { +static void doUpdateExprColumnIndex(SQuery* pQuery) { + assert(pQuery->pSelectExpr != NULL && pQuery != NULL); +// int32_t i = 0, j = 0; +// while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) { +// if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) { +// pQuery->colList[i++].colIdx = (int16_t)j++; +// } else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) { +// pQuery->colList[i++].colIdx = -1; +// } else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) { +// j++; +// } +// } + +// while (i < pQuery->numOfCols) { +// pQuery->colList[i++].colIdx = -1; // not such column in current meter +// } + + for(int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + SSqlFuncExprMsg* pSqlExprMsg = &pQuery->pSelectExpr[k].pBase; + if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.flag == TSDB_COL_TAG) { + continue; + } + + SColIndexEx* pColIndexEx = &pSqlExprMsg->colInfo; + for(int32_t f = 0; f < pQuery->numOfCols; ++f) { + if (pColIndexEx->colId == pQuery->colList[f].info.colId) { + pColIndexEx->colIdx = f; + break; + } + } + } +} + +static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pExprs, + SArray *pTableIdList) { SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); if (pQInfo == NULL) { goto _clean_memory; @@ -5944,20 +5765,21 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou if (pQuery->colList == NULL) { goto _clean_memory; } - + for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i].info = pQueryMsg->colList[i]; -// SColumnInfo *pColInfo = &pQuery->colList[i].data; -// pColInfo->filters = NULL; -// if (colList[i].numOfFilters > 0) { -// pColInfo->filters = calloc(1, colList[i].numOfFilters * sizeof(SColumnFilterInfo)); -// -// for (int32_t j = 0; j < colList[i].numOfFilters; ++j) { -// tscColumnFilterInfoCopy(&pColInfo->filters[j], &colList[i].filters[j]); -// } -// } else { -// pQuery->colList[i].data.filters = NULL; -// } + + SColumnInfo *pColInfo = &pQuery->colList[i].info; + pColInfo->filters = NULL; + // if (colList[i].numOfFilters > 0) { + // pColInfo->filters = calloc(1, colList[i].numOfFilters * sizeof(SColumnFilterInfo)); + // + // for (int32_t j = 0; j < colList[i].numOfFilters; ++j) { + // tscColumnFilterInfoCopy(&pColInfo->filters[j], &colList[i].filters[j]); + // } + // } else { + // pQuery->colList[i].data.filters = NULL; + // } } // calculate the result row size @@ -5965,6 +5787,8 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou assert(pExprs[col].resBytes > 0); pQuery->rowSize += pExprs[col].resBytes; } + + doUpdateExprColumnIndex(pQuery); int32_t ret = vnodeCreateFilterInfo(pQInfo, pQuery); if (ret != TSDB_CODE_SUCCESS) { @@ -5972,18 +5796,20 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } // prepare the result buffer - pQuery->sdata = (SData **)calloc(pQuery->numOfOutputCols, sizeof(SData *)); + pQuery->sdata = (SData **)calloc(pQuery->numOfOutputCols, POINTER_BYTES); if (pQuery->sdata == NULL) { goto _clean_memory; } // set the output buffer capacity - pQuery->capacity = 4096; + pQuery->rec.capacity = 4096; + pQuery->rec.threshold = 2; + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { assert(pExprs[col].interResBytes >= pExprs[col].resBytes); // allocate additional memory for interResults that are usually larger then final results - size_t size = (pQuery->capacity + 1) * pExprs[col].resBytes + pExprs[col].interResBytes + sizeof(SData); + size_t size = (pQuery->rec.capacity + 1) * pExprs[col].resBytes + pExprs[col].interResBytes + sizeof(SData); pQuery->sdata[col] = (SData *)calloc(1, size); if (pQuery->sdata[col] == NULL) { goto _clean_memory; @@ -6001,12 +5827,23 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } // to make sure third party won't overwrite this structure - pQInfo->signature = (uint64_t)pQInfo; + pQInfo->signature = pQInfo; pQInfo->pTableIdList = pTableIdList; - - pQuery->pos = -1; - // dTrace("vid:%d sid:%d meterId:%s, QInfo is allocated:%p", pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQInfo); + pQuery->pos = -1; + + pQuery->window.skey = pQueryMsg->window.skey; + pQuery->window.ekey = pQueryMsg->window.ekey; + pQuery->lastKey = pQuery->window.skey; + + if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { + dError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); + goto _clean_memory; + } + + vnodeParametersSafetyCheck(pQuery); + + dTrace("qmsg:%p create QInfo:%p, QInfo created", pQueryMsg, pQInfo); return pQInfo; _clean_memory: @@ -6030,12 +5867,12 @@ _clean_memory: return NULL; } -bool isQInfoValid(void *param) { +static bool isValidQInfo(void *param) { SQInfo *pQInfo = (SQInfo *)param; if (pQInfo == NULL) { return false; } - + /* * pQInfo->signature may be changed by another thread, so we assign value of signature * into local variable, then compare by using local variable @@ -6044,31 +5881,66 @@ bool isQInfoValid(void *param) { return (sig == (uint64_t)pQInfo); } -void vnodeFreeQInfo(SQInfo* pQInfo, bool decQueryRef) { - if (!isQInfoValid(pQInfo)) { - return; - } +static void freeQInfo(SQInfo *pQInfo); +static int32_t initializeQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { + int32_t code = TSDB_CODE_SUCCESS; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - pQInfo->killed = 1; - dTrace("QInfo:%p start to free SQInfo", pQInfo); + STSBuf *pTSBuf = NULL; + if (pQueryMsg->tsLen > 0) { // open new file to save the result + char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; + pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); + + tsBufResetPos(pTSBuf); + tsBufNextPos(pTSBuf); + } - if (decQueryRef) { - vnodeDecMeterRefcnt(pQInfo); + // only the successful complete requries the sem_post/over = 1 operations. + if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || + (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { + dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, + pQuery->window.ekey, pQuery->order.order); + + sem_post(&pQInfo->dataReady); + setQueryStatus(pQuery, QUERY_COMPLETED); + return TSDB_CODE_SUCCESS; + } + + // filter the qualified + if ((code = doInitializeQInfo(pQInfo, pTSBuf, tsdb, isSTable)) != TSDB_CODE_SUCCESS) { + goto _error; + } + + // dTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, + // pQInfo->refCount); + return code; + +_error: + // table query ref will be decrease during error handling + freeQInfo(pQInfo); + return code; +} + +static void freeQInfo(SQInfo *pQInfo) { + if (!isValidQInfo(pQInfo)) { + return; } - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + setQueryKilled(pQInfo); - for (int col = 0; col < pQuery->numOfOutputCols; ++col) { + dTrace("QInfo:%p start to free SQInfo", pQInfo); + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { tfree(pQuery->sdata[col]); } -// for (int col = 0; col < pQuery->numOfCols; ++col) { -// vnodeFreeColumnInfo(&pQuery->colList[col].data); -// } -// -// if (pQuery->colList[0].colIdx != PRIMARYKEY_TIMESTAMP_COL_INDEX) { -// tfree(pQuery->tsData); -// } + // for (int col = 0; col < pQuery->numOfCols; ++col) { + // vnodeFreeColumnInfo(&pQuery->colList[col].data); + // } + // + // if (pQuery->colList[0].colIdx != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + // tfree(pQuery->tsData); + // } sem_destroy(&(pQInfo->dataReady)); vnodeQueryFreeQInfoEx(pQInfo); @@ -6104,159 +5976,252 @@ void vnodeFreeQInfo(SQInfo* pQInfo, bool decQueryRef) { tfree(pQuery->pGroupbyExpr); tfree(pQuery); -// dTrace("QInfo:%p vid:%d sid:%d meterId:%s, QInfo is freed", pQInfo, pObj->vnode, pObj->sid, pObj->meterId); + dTrace("QInfo:%p QInfo is freed", pQInfo); - //destroy signature, in order to avoid the query process pass the object safety check + // destroy signature, in order to avoid the query process pass the object safety check memset(pQInfo, 0, sizeof(SQInfo)); tfree(pQInfo); } -static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, - SArray* pTableIdList, SQInfo **pQInfo) { - int32_t code = TSDB_CODE_SUCCESS; - - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pSqlExprs, pTableIdList); - if (pQInfo == NULL) { - code = TSDB_CODE_SERV_OUT_OF_MEMORY; - goto _error; - } - - SQuery* pQuery = (*pQInfo)->runtimeEnv.pQuery; - dTrace("qmsg:%p create QInfo:%p, QInfo created", pQueryMsg, pQInfo); - -// STableIdInfo **pTableIdList = (STableIdInfo **)pQueryMsg->pSidExtInfo; -// if (pTableIdList != NULL && pTableIdList[0]->key > 0) { -// pQuery->window.skey = pTableIdList[0]->key; -// } else { - pQuery->window.skey = pQueryMsg->window.skey; - pQuery->window.ekey = pQueryMsg->window.ekey; +static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - pQuery->lastKey = pQuery->window.skey; - - if (sem_init(&(*pQInfo)->dataReady, 0, 0) != 0) { - // dError("QInfo:%p vid:%d sid:%d meterId:%s, init dataReady sem failed, reason:%s", pQInfo, pMeterObj->vnode, - // pMeterObj->sid, pMeterObj->meterId, strerror(errno)); - code = TSDB_CODE_APP_ERROR; - goto _error; - } - - vnodeParametersSafetyCheck(pQuery); - - STSBuf *pTSBuf = NULL; - if (pQueryMsg->tsLen > 0) { // open new file to save the result - char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; - pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); - - tsBufResetPos(pTSBuf); - tsBufNextPos(pTSBuf); + /* + * get the file size and set the numOfRows to be the file size, since for tsComp query, + * the returned row size is equalled to 1 + * TODO handle the case that the file is too large to send back one time + */ + if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { + struct stat fstat; + if (stat(pQuery->sdata[0]->data, &fstat) == 0) { + *numOfRows = fstat.st_size; + return fstat.st_size; + } else { + dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); + return 0; + } + } else { + return pQuery->rowSize * (*numOfRows); } +} - if ((code = vnodeQueryTablePrepare(*pQInfo, pTSBuf)) != TSDB_CODE_SUCCESS) { - goto _error; +static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { + // the remained number of retrieved rows, not the interpolated result + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + // load data from file to msg buffer + if (isTSCompQuery(pQuery)) { + int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); + + // make sure file exist + if (FD_VALID(fd)) { + size_t s = lseek(fd, 0, SEEK_END); + dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); + + lseek(fd, 0, SEEK_SET); + read(fd, data, s); + close(fd); + + unlink(pQuery->sdata[0]->data); + } else { + dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, + pQuery->sdata[0]->data, strerror(errno)); + } + } else { + doCopyQueryResultToMsg(pQInfo, pQuery->rec.size, data); } - - // if (pQInfo->over == 1) { - // vnodeAddRefCount(pQInfo); // for retrieve procedure - // return pQInfo; - // } - - // dTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, - // pQInfo->refCount); - return code; -_error: - // table query ref will be decrease during error handling - vnodeFreeQInfo(*pQInfo, false); - return code; + pQuery->rec.total += pQuery->rec.size; + dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + + return TSDB_CODE_SUCCESS; + + // todo if interpolation exists, the result may be dump to client by several rounds } -int32_t qCreateQueryInfo(SQueryTableMsg *pQueryTableMsg, SQInfo **pQInfo) { - assert(pQueryTableMsg != NULL); +int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) { + assert(pQueryMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; - SArray* pTableIdList = NULL; - if ((code = convertQueryTableMsg(pQueryTableMsg, &pTableIdList)) != TSDB_CODE_SUCCESS) { + + SArray *pTableIdList = NULL; + SSqlFuncExprMsg** pExprMsg = NULL; + wchar_t* tagCond = NULL; + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond)) != TSDB_CODE_SUCCESS) { return code; } - if (pQueryTableMsg->numOfTables <= 0) { - dError("Invalid number of tables to query, numOfTables:%d", pQueryTableMsg->numOfTables); + if (pQueryMsg->numOfTables <= 0) { + dError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); code = TSDB_CODE_INVALID_QUERY_MSG; goto _query_over; } // todo check vnode status if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) { - dError("qmsg:%p, SQueryTableMsg wrong format", pQueryTableMsg); + dError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); code = TSDB_CODE_INVALID_QUERY_MSG; goto _query_over; } SSqlFunctionExpr *pExprs = NULL; - if ((code = createSqlFunctionExprFromMsg(pQueryTableMsg, &pExprs)) != TSDB_CODE_SUCCESS) { + if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { goto _query_over; } - SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryTableMsg, &code); - if ((pGroupbyExpr == NULL && pQueryTableMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { + SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, &code); + if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { goto _query_over; } - - if (QUERY_IS_STABLE_QUERY(pQueryTableMsg->queryType)) { - // pObj->qhandle = vnodeQueryOnMultiMeters(pMeterObjList, pGroupbyExpr, pExprs, pQueryTableMsg, &code); + + // super table query + SArray* res = NULL; + bool isSTableQuery = false; + if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { + isSTableQuery = true; + + STableId* id = taosArrayGet(pTableIdList, 0); + id->uid = -1; + + res = tsdbQueryTableList(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen); + if (taosArrayGetSize(res) == 0) { // no qualified table in stable query in this vnode + code = TSDB_CODE_SUCCESS; + goto _query_over; + } } else { - code = createQInfo(pQueryTableMsg, pGroupbyExpr, pExprs, pTableIdList, pQInfo); + res = pTableIdList; } - + + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, res); + if ((*pQInfo) == NULL) { + code = TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + code = initializeQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); + _query_over: if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pTableIdList); } - + // if failed to add ref for all meters in this query, abort current query // if (code != TSDB_CODE_SUCCESS) { - // vnodeDecQueryRefCount(pQueryTableMsg, pMeterObjList, incNumber); + // vnodeDecQueryRefCount(pQueryMsg, pMeterObjList, incNumber); // } // - // tfree(pQueryTableMsg->pSqlFuncExprs); + // tfree(pQueryMsg->pSqlFuncExprs); // tfree(pMeterObjList); // ret = vnodeSendQueryRspMsg(pObj, code, pObj->qhandle); // - // tfree(pQueryTableMsg->pSidExtInfo); - // for(int32_t i = 0; i < pQueryTableMsg->numOfCols; ++i) { - // vnodeFreeColumnInfo(&pQueryTableMsg->colList[i]); + // tfree(pQueryMsg->pSidExtInfo); + // for(int32_t i = 0; i < pQueryMsg->numOfCols; ++i) { + // vnodeFreeColumnInfo(&pQueryMsg->colList[i]); // } // // atomic_fetch_add_32(&vnodeSelectReqNum, 1); return TSDB_CODE_SUCCESS; } -int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo, int32_t *numOfRows, int32_t* rowsize) { - if (pQInfo == NULL || !isQInfoValid(pQInfo)) { +void qTableQuery(SQInfo *pQInfo) { + if (pQInfo == NULL || pQInfo->signature != pQInfo) { + dTrace("%p freed abort query", pQInfo); + return; + } + + if (isQueryKilled(pQInfo)) { + dTrace("QInfo:%p it is already killed, abort", pQInfo); + return; + } + + dTrace("QInfo:%p query task is launched", pQInfo); + + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + if (numOfTables == 1) { + singleTableQueryImpl(pQInfo); + } else { + multiTableQueryImpl(pQInfo); + } + + // vnodeDecRefCount(pQInfo); +} + +int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo) { + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } - if (pQInfo->killed) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); if (pQInfo->code == TSDB_CODE_SUCCESS) { return TSDB_CODE_QUERY_CANCELLED; - } else { // in case of not TSDB_CODE_SUCCESS, return the code to client - return abs(pQInfo->code); + } else { // in case of not TSDB_CODE_SUCCESS, return the code to client + return (pQInfo->code >= 0)? pQInfo->code:(-pQInfo->code); } } - + sem_wait(&pQInfo->dataReady); + dTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.size, + pQInfo->code); + + return (pQInfo->code >= 0)? pQInfo->code:(-pQInfo->code); +} + +bool qHasMoreResultsToRetrieve(SQInfo* pQInfo) { + if (pQInfo == NULL || pQInfo->signature != pQInfo || pQInfo->code != TSDB_CODE_SUCCESS) { + return false; + } + + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { + return false; + } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { + return true; + } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + return true; + } else { + assert(0); + } +} + +int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen) { + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_INVALID_QHANDLE; + } SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + size_t size = getResultSize(pQInfo, &pQuery->rec.size); + *contLen = size + sizeof(SRetrieveTableRsp); -// *numOfRows = pQInfo->rec.pointsRead; -// *rowsize = pQuery->rowSize; - *numOfRows = 1; + // todo handle failed to allocate memory + *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); + (*pRsp)->numOfRows = htonl(pQuery->rec.size); + + int32_t code = pQInfo->code; + if (code == TSDB_CODE_SUCCESS) { + (*pRsp)->offset = htobe64(pQuery->limit.offset); + (*pRsp)->useconds = htobe64(pQInfo->elapsedTime); + } else { + (*pRsp)->offset = 0; + (*pRsp)->useconds = 0; + } -// dTrace("QInfo:%p, retrieve data info completed, precision:%d, rowsize:%d, rows:%d, code:%d", pQInfo, *timePrec, -// *rowsize, *numOfRows, pQInfo->code); + if (pQuery->rec.size > 0 && code == TSDB_CODE_SUCCESS) { + code = doDumpQueryResult(pQInfo, (*pRsp)->data); + } else { + code = pQInfo->code; + } - if (pQInfo->code < 0) { // less than 0 means there are error existed. - return -pQInfo->code; + if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { + (*pRsp)->completed = 1; // notify no more result to client + freeQInfo(pQInfo); } -} + + return code; + +// if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { +// dTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); +// vnodeDecRefCount(pObj->qhandle); +// pObj->qhandle = NULL; +// } +} \ No newline at end of file diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0ae86007568d3d7a00ba6f7b147ac5f0d00ee74a --- /dev/null +++ b/src/query/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) + +IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) + MESSAGE(STATUS "gTest library found, build unit test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) +ENDIF() \ No newline at end of file diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c23f0f59241d7bbc7527f7fe332fe97b666acbcc --- /dev/null +++ b/src/query/tests/histogramTest.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "tstoken.h" +#include "tutil.h" + +#include "qhistogram.h" + +/* test validate the names for table/database */ +TEST(testCase, histogram_binary_search) { + SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); + + pHisto->numOfEntries = 10; + for (int32_t i = 0; i < 10; ++i) { + pHisto->elems[i].num = 1; + pHisto->elems[i].val = i; + } + + int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1); + assert(idx == 1); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9); + assert(idx == 9); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20); + assert(idx == 10); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1); + assert(idx == 0); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); + assert(idx == 4); + + free(pHisto); +} + +TEST(testCase, histogram_add) { + SHistogramInfo* pHisto = NULL; + + /** + * use arrayList, elapsed time is: + * before: + * 10,000,000 45sec, bin:1000 (-O0) / 17sec. bin:1000, (-O3) + * + * after: + * + */ + struct timeval systemTime; + gettimeofday(&systemTime, NULL); + int64_t st = + (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; + for (int32_t i = 0; i < 10000; ++i) { + tHistogramAdd(&pHisto, i); + // tHistogramPrint(pHisto); + } + // + gettimeofday(&systemTime, NULL); + int64_t et = + (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; + printf("total elapsed time: %ld\n", et - st); + + printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries); + tHistogramPrint(pHisto); + + printf("%ld\n", tHistogramSum(pHisto, 1.5)); + printf("%ld\n", tHistogramSum(pHisto, 2)); + printf("%ld\n", tHistogramSum(pHisto, 3)); + printf("%ld\n", tHistogramSum(pHisto, 4)); + printf("%ld\n", tHistogramSum(pHisto, 5)); + printf("%ld\n", tHistogramSum(pHisto, 6)); + + for (int32_t i = 399; i < 400; ++i) { + printf("val:%d, %ld\n", i, tHistogramSum(pHisto, i)); + } + + double ratio[] = {0 / 100, 20.0 / 100, 88.0 / 100, 100 / 100}; + double* res = tHistogramUniform(pHisto, ratio, 4); + for (int32_t i = 0; i < 4; ++i) { + printf("%f\n", res[i]); + } + + SHistogramInfo* pHisto1 = NULL; + for (int32_t i = (90000 - 1); i >= 80000; --i) { + tHistogramAdd(&pHisto1, i); + } + tHistogramPrint(pHisto1); + + SHistogramInfo* pRes = tHistogramMerge(pHisto1, pHisto, MAX_HISTOGRAM_BIN); + assert(pRes->numOfElems == pHisto->numOfElems + pHisto1->numOfElems); + tHistogramPrint(pRes); + + tHistogramDestroy(&pHisto); + tHistogramDestroy(&pHisto1); + tHistogramDestroy(&pRes); + free(res); +} + +TEST(testCase, heapsort) { + // int32_t num = 20; + // + // SHeapEntry* pEntry = tHeapCreate(num); + // + // for(int32_t i=0; i +#include +#include +#include + +#include "tsqlfunction.h" + +TEST(testCase, patternMatchTest) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + const char* str = "abcdef"; + int32_t ret = patternMatch("a%b%", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tm01"; + ret = patternMatch("tm__", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tkm1"; + ret = patternMatch("t%m1", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tkm1"; + ret = patternMatch("%m1", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = ""; + ret = patternMatch("%_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = "1"; + ret = patternMatch("%__", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = ""; + ret = patternMatch("%", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = " "; + ret = patternMatch("_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "!"; + ret = patternMatch("%_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefg"; + ret = patternMatch("abc%fg", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%fg", str, 7, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%f_", str, 6, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%f_", str, 1, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("ab", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("a%", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("a__", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); +} diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fca9ac8f9d82911975dc9e917b48b93be26142de --- /dev/null +++ b/src/query/tests/resultBufferTest.cpp @@ -0,0 +1,35 @@ +#include +#include +#include + +#include "taos.h" +#include "qresultBuf.h" +#include "tsdb.h" + +namespace { +// simple test +void simpleTest() { + SDiskbasedResultBuf* pResultBuf = NULL; + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64); + + int32_t pageId = 0; + int32_t groupId = 0; + + tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + ASSERT_TRUE(pBufPage != NULL); + + ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64); + ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L); + + SIDList list = getDataBufPagesIdList(pResultBuf, groupId); + ASSERT_EQ(list.size, 1); + + ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); + + destroyResultBuf(pResultBuf); +} +} // namespace + +TEST(testCase, resultBufferTest) { + simpleTest(); +} diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2392bd00b53c52126585d5bf1e795fe8db15f106 --- /dev/null +++ b/src/query/tests/tsBufTest.cpp @@ -0,0 +1,451 @@ +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "tstoken.h" +#include "ttime.h" +#include "tutil.h" +#include "qtsbuf.h" + +namespace { +/** + * + * @param num total number + * @param step gap between two consecutive ts + * @return + */ +int64_t* createTsList(int32_t num, int64_t start, int32_t step) { + int64_t* pList = (int64_t*)malloc(num * sizeof(int64_t)); + + for (int64_t i = 0; i < num; ++i) { + pList[i] = start + i * step; + } + + return pList; +} + +// simple test +void simpleTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + // write 10 ts points + int32_t num = 10; + int64_t tag = 1; + + int64_t* list = createTsList(10, 10000000, 30); + tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); + EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +// one large list of ts, the ts list need to be split into several small blocks +void largeTSTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + // write 10 ts points + int32_t num = 1000000; + int64_t tag = 1; + + int64_t* list = createTsList(num, 10000000, 30); + tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + + // the data has been flush to disk, no data in cache + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void multiTagsTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t tag = 1; + int64_t start = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void multiVnodeTagsTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t start = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + + // 2000 vnodes + for (int32_t j = 0; j < 20; ++j) { + // vnodeId:0 + start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void loadDataTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t oldStart = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + int32_t numOfVnode = 200; + + // 10000 vnodes + for (int32_t j = 0; j < numOfVnode; ++j) { + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %lld\n", i, list[0]); + + free(list); + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + // create from exists file + STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false); + EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder); + EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode); + EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize); + + EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset); + EXPECT_EQ(pNewBuf->pData[0].info.numOfBlocks, pTSBuf->pData[0].info.numOfBlocks); + EXPECT_EQ(pNewBuf->pData[0].info.compLen, pTSBuf->pData[0].info.compLen); + + EXPECT_STREQ(pNewBuf->path, pTSBuf->path); + + tsBufResetPos(pNewBuf); + + int64_t s = taosGetTimestampUs(); + printf("start:%lld\n", s); + + int32_t x = 0; + while (tsBufNextPos(pNewBuf)) { + STSElem elem = tsBufGetElem(pNewBuf); + if (++x == 100000000) { + break; + } + + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + int64_t e = taosGetTimestampUs(); + printf("end:%lld, elapsed:%lld, total obj:%d\n", e, e - s, x); +} + +void randomIncTsTest() {} + +void TSTraverse() { + // 10000 vnodes + int32_t num = 200000; + int64_t oldStart = 10000000; + int32_t numOfTags = 3; + int32_t step = 30; + int32_t numOfVnode = 2; + + STSBuf* pTSBuf = tsBufCreate(true); + + for (int32_t j = 0; j < numOfVnode; ++j) { + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]); + + free(list); + start += step * num; + + list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + tsBufResetPos(pTSBuf); + + //////////////////////////////////////////////////////////////////////////////////////// + // reverse traverse + int64_t s = taosGetTimestampUs(); + printf("start:%lld\n", s); + + pTSBuf->cur.order = TSQL_SO_DESC; + + // complete reverse traverse + int32_t x = 0; + while (tsBufNextPos(pTSBuf)) { + STSElem elem = tsBufGetElem(pTSBuf); + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + // specify the data block with vnode and tags value + tsBufResetPos(pTSBuf); + pTSBuf->cur.order = TSQL_SO_DESC; + + int32_t startVnode = 1; + int32_t startTag = 2; + + tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + + int32_t totalOutput = 10; + while (1) { + STSElem elem = tsBufGetElem(pTSBuf); + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + + if (!tsBufNextPos(pTSBuf)) { + break; + } + + if (--totalOutput <= 0) { + totalOutput = 10; + + tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + + if (startTag == 0) { + startVnode -= 1; + startTag = 3; + } + + if (startVnode < 0) { + break; + } + } + } + + ///////////////////////////////////////////////////////////////////////////////// + // traverse + pTSBuf->cur.order = TSQL_SO_ASC; + tsBufResetPos(pTSBuf); + + // complete forwards traverse + while (tsBufNextPos(pTSBuf)) { + STSElem elem = tsBufGetElem(pTSBuf); + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + // specify the data block with vnode and tags value + tsBufResetPos(pTSBuf); + pTSBuf->cur.order = TSQL_SO_ASC; + + startVnode = 1; + startTag = 2; + + tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + + totalOutput = 10; + while (1) { + STSElem elem = tsBufGetElem(pTSBuf); + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + + if (!tsBufNextPos(pTSBuf)) { + break; + } + + if (--totalOutput <= 0) { + totalOutput = 10; + + tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + + if (startTag < 0) { + startVnode -= 1; + startTag = 3; + } + + if (startVnode < 0) { + break; + } + } + } +} + +void performanceTest() {} + +void emptyTagTest() {} + +void invalidFileTest() { + const char* cmd = "touch /tmp/test"; + + // create empty file + system(cmd); + + STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true); + EXPECT_TRUE(pNewBuf == NULL); + + pNewBuf = tsBufCreateFromFile("/tmp/911", true); + EXPECT_TRUE(pNewBuf == NULL); +} + +void mergeDiffVnodeBufferTest() { + STSBuf* pTSBuf1 = tsBufCreate(true); + STSBuf* pTSBuf2 = tsBufCreate(true); + + int32_t step = 30; + int32_t num = 1000; + int32_t numOfTags = 10; + + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t)); + + free(list); + + start += step * num; + } + + tsBufFlush(pTSBuf2); + + tsBufMerge(pTSBuf1, pTSBuf2, 9); + EXPECT_EQ(pTSBuf1->numOfVnodes, 2); + EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); + + tsBufDisplay(pTSBuf1); + + tsBufDestory(pTSBuf2); + tsBufDestory(pTSBuf1); +} + +void mergeIdenticalVnodeBufferTest() { + STSBuf* pTSBuf1 = tsBufCreate(true); + STSBuf* pTSBuf2 = tsBufCreate(true); + + int32_t step = 30; + int32_t num = 1000; + int32_t numOfTags = 10; + + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + + tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { + int64_t* list = createTsList(num, start, step); + + tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + tsBufFlush(pTSBuf2); + + tsBufMerge(pTSBuf1, pTSBuf2, 12); + EXPECT_EQ(pTSBuf1->numOfVnodes, 1); + EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); + + tsBufResetPos(pTSBuf1); + while (tsBufNextPos(pTSBuf1)) { + STSElem elem = tsBufGetElem(pTSBuf1); + EXPECT_EQ(elem.vnode, 12); + + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + tsBufDestory(pTSBuf1); + tsBufDestory(pTSBuf2); +} +} // namespace + +TEST(testCase, tsBufTest) { + simpleTest(); + largeTSTest(); + multiTagsTest(); + multiVnodeTagsTest(); + loadDataTest(); + invalidFileTest(); + // randomIncTsTest(); + TSTraverse(); + mergeDiffVnodeBufferTest(); + mergeIdenticalVnodeBufferTest(); +} diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c7520c90f9fbdb818d85d8f3b321c49820441f2d --- /dev/null +++ b/src/query/tests/unitTest.cpp @@ -0,0 +1,769 @@ +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "../../client/inc/tscUtil.h" +#include "ttime.h" +#include "tutil.h" +#include "tvariant.h" +#include "ttokendef.h" + +namespace { +int32_t testValidateName(char* name) { + SSQLToken token = {0}; + token.z = name; + token.n = strlen(name); + token.type = 0; + + tSQLGetToken(name, &token.type); + return tscValidateName(&token); +} +} + +static void _init_tvariant_bool(tVariant* t) { + t->i64Key = TSDB_FALSE; + t->nType = TSDB_DATA_TYPE_BOOL; +} + +static void _init_tvariant_tinyint(tVariant* t) { + t->i64Key = -27; + t->nType = TSDB_DATA_TYPE_TINYINT; +} + +static void _init_tvariant_int(tVariant* t) { + t->i64Key = -23997659; + t->nType = TSDB_DATA_TYPE_INT; +} + +static void _init_tvariant_bigint(tVariant* t) { + t->i64Key = -3333333333333; + t->nType = TSDB_DATA_TYPE_BIGINT; +} + +static void _init_tvariant_float(tVariant* t) { + t->dKey = -8991212199.8987878776; + t->nType = TSDB_DATA_TYPE_FLOAT; +} + +static void _init_tvariant_binary(tVariant* t) { + tVariantDestroy(t); + + t->pz = (char*)calloc(1, 20); //"2e3"); + t->nType = TSDB_DATA_TYPE_BINARY; + strcpy(t->pz, "2e5"); + t->nLen = strlen(t->pz); +} + +static void _init_tvariant_nchar(tVariant* t) { + tVariantDestroy(t); + + t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE); + t->nType = TSDB_DATA_TYPE_NCHAR; + wcscpy(t->wpz, L"-2000000.8765"); + t->nLen = wcslen(t->wpz); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +/* test validate the names for table/database */ +TEST(testCase, db_table_name) { + + char t01[] = "abc"; + EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS); + + char t02[] = "'abc'"; + EXPECT_EQ(testValidateName(t02), TSDB_CODE_SUCCESS); + + char t1[] = "abc.def"; + EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS); + printf("%s\n", t1); + + char t2[] = "'abc.def'"; + EXPECT_EQ(testValidateName(t2), TSDB_CODE_SUCCESS); + printf("%s\n", t2); + + char t3[] = "'abc'.def"; + EXPECT_EQ(testValidateName(t3), TSDB_CODE_SUCCESS); + printf("%s\n", t3); + + char t4[] = "'abc'.'def'"; + EXPECT_EQ(testValidateName(t4), TSDB_CODE_SUCCESS); + + char t5[] = "table.'def'"; + EXPECT_EQ(testValidateName(t5), TSDB_CODE_INVALID_SQL); + + char t6[] = "'table'.'def'"; + EXPECT_EQ(testValidateName(t6), TSDB_CODE_INVALID_SQL); + + char t7[] = "'_ab1234'.'def'"; + EXPECT_EQ(testValidateName(t7), TSDB_CODE_SUCCESS); + printf("%s\n", t7); + + char t8[] = "'_ab&^%1234'.'def'"; + EXPECT_EQ(testValidateName(t8), TSDB_CODE_INVALID_SQL); + + char t9[] = "'_123'.'gtest中文'"; + EXPECT_EQ(testValidateName(t9), TSDB_CODE_INVALID_SQL); + + char t10[] = "abc.'gtest中文'"; + EXPECT_EQ(testValidateName(t10), TSDB_CODE_INVALID_SQL); + + char t10_1[] = "abc.'中文gtest'"; + EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_INVALID_SQL); + + char t11[] = "'192.168.0.1'.abc"; + EXPECT_EQ(testValidateName(t11), TSDB_CODE_INVALID_SQL); + + char t12[] = "192.168.0.1.abc"; + EXPECT_EQ(testValidateName(t12), TSDB_CODE_INVALID_SQL); + + char t13[] = "abc."; + EXPECT_EQ(testValidateName(t13), TSDB_CODE_INVALID_SQL); + + char t14[] = ".abc"; + EXPECT_EQ(testValidateName(t14), TSDB_CODE_INVALID_SQL); + + char t15[] = ".'abc'"; + EXPECT_EQ(testValidateName(t15), TSDB_CODE_INVALID_SQL); + + char t16[] = ".abc'"; + EXPECT_EQ(testValidateName(t16), TSDB_CODE_INVALID_SQL); + + char t17[] = "123a.\"abc\""; + EXPECT_EQ(testValidateName(t17), TSDB_CODE_INVALID_SQL); + printf("%s\n", t17); + + char t18[] = "a.\"abc\""; + EXPECT_EQ(testValidateName(t18), TSDB_CODE_SUCCESS); + printf("%s\n", t18); + + char t19[] = "'_ab1234'.'def'.'ab123'"; + EXPECT_EQ(testValidateName(t19), TSDB_CODE_INVALID_SQL); + + char t20[] = "'_ab1234*&^'"; + EXPECT_EQ(testValidateName(t20), TSDB_CODE_INVALID_SQL); + + char t21[] = "'1234_abc'"; + EXPECT_EQ(testValidateName(t21), TSDB_CODE_INVALID_SQL); + + + // =======Containing capital letters================= + char t30[] = "ABC"; + EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS); + + char t31[] = "'ABC'"; + EXPECT_EQ(testValidateName(t31), TSDB_CODE_SUCCESS); + + char t32[] = "ABC.def"; + EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS); + + char t33[] = "'ABC.def"; + EXPECT_EQ(testValidateName(t33), TSDB_CODE_INVALID_SQL); + + char t33_0[] = "abc.DEF'"; + EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_INVALID_SQL); + + char t34[] = "'ABC.def'"; + //int32_t tmp0 = testValidateName(t34); + EXPECT_EQ(testValidateName(t34), TSDB_CODE_SUCCESS); + + char t35[] = "'ABC'.def"; + EXPECT_EQ(testValidateName(t35), TSDB_CODE_SUCCESS); + + char t36[] = "'ABC'.'DEF'"; + EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS); + + char t37[] = "abc.'DEF'"; + EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS); + + char t37_1[] = "abc.'_123DEF'"; + EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS); + + char t38[] = "'abc'.'DEF'"; + EXPECT_EQ(testValidateName(t38), TSDB_CODE_SUCCESS); + + // do not use key words + char t39[] = "table.'DEF'"; + EXPECT_EQ(testValidateName(t39), TSDB_CODE_INVALID_SQL); + + char t40[] = "'table'.'DEF'"; + EXPECT_EQ(testValidateName(t40), TSDB_CODE_INVALID_SQL); + + char t41[] = "'_abXYZ1234'.'deFF'"; + EXPECT_EQ(testValidateName(t41), TSDB_CODE_SUCCESS); + + char t42[] = "'_abDEF&^%1234'.'DIef'"; + EXPECT_EQ(testValidateName(t42), TSDB_CODE_INVALID_SQL); + + char t43[] = "'_123'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t43), TSDB_CODE_INVALID_SQL); + + char t44[] = "'aABC'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t44), TSDB_CODE_INVALID_SQL); + + char t45[] = "'ABC'."; + EXPECT_EQ(testValidateName(t45), TSDB_CODE_INVALID_SQL); + + char t46[] = ".'ABC'"; + EXPECT_EQ(testValidateName(t46), TSDB_CODE_INVALID_SQL); + + char t47[] = "a.\"aTWc\""; + EXPECT_EQ(testValidateName(t47), TSDB_CODE_SUCCESS); + + // ================has space ================= + char t60[] = " ABC "; + EXPECT_EQ(testValidateName(t60), TSDB_CODE_INVALID_SQL); + + char t60_1[] = " ABC "; + EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_INVALID_SQL); + + char t61[] = "' ABC '"; + EXPECT_EQ(testValidateName(t61), TSDB_CODE_SUCCESS); + + char t61_1[] = "' ABC '"; + EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_SUCCESS); + + char t62[] = " ABC . def "; + EXPECT_EQ(testValidateName(t62), TSDB_CODE_INVALID_SQL); + + char t63[] = "' ABC . def "; + EXPECT_EQ(testValidateName(t63), TSDB_CODE_INVALID_SQL); + + char t63_0[] = " abc . DEF ' "; + EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_INVALID_SQL); + + char t64[] = " ' ABC . def ' "; + //int32_t tmp1 = testValidateName(t64); + EXPECT_EQ(testValidateName(t64), TSDB_CODE_INVALID_SQL); + + char t65[] = " ' ABC '. def "; + EXPECT_EQ(testValidateName(t65), TSDB_CODE_INVALID_SQL); + + char t66[] = "' ABC '.' DEF '"; + EXPECT_EQ(testValidateName(t66), TSDB_CODE_SUCCESS); + + char t67[] = "abc . ' DEF '"; + EXPECT_EQ(testValidateName(t67), TSDB_CODE_INVALID_SQL); + + char t68[] = "' abc '.' DEF '"; + EXPECT_EQ(testValidateName(t68), TSDB_CODE_SUCCESS); + + // do not use key words + char t69[] = "table.'DEF'"; + EXPECT_EQ(testValidateName(t69), TSDB_CODE_INVALID_SQL); + + char t70[] = "'table'.'DEF'"; + EXPECT_EQ(testValidateName(t70), TSDB_CODE_INVALID_SQL); + + char t71[] = "'_abXYZ1234 '.' deFF '"; + EXPECT_EQ(testValidateName(t71), TSDB_CODE_SUCCESS); + + char t72[] = "'_abDEF&^%1234'.' DIef'"; + EXPECT_EQ(testValidateName(t72), TSDB_CODE_INVALID_SQL); + + char t73[] = "'_123'.' Gtest中文'"; + EXPECT_EQ(testValidateName(t73), TSDB_CODE_INVALID_SQL); + + char t74[] = "' aABC'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t74), TSDB_CODE_INVALID_SQL); + + char t75[] = "' ABC '."; + EXPECT_EQ(testValidateName(t75), TSDB_CODE_INVALID_SQL); + + char t76[] = ".' ABC'"; + EXPECT_EQ(testValidateName(t76), TSDB_CODE_INVALID_SQL); + + char t77[] = " a . \"aTWc\" "; + EXPECT_EQ(testValidateName(t77), TSDB_CODE_INVALID_SQL); + + char t78[] = " a.\"aTWc \""; + EXPECT_EQ(testValidateName(t78), TSDB_CODE_INVALID_SQL); + + + // ===============muti string by space =================== + // There's no such case. + //char t160[] = "A BC"; + //EXPECT_EQ(testValidateName(t160), TSDB_CODE_INVALID_SQL); + //printf("end:%s\n", t160); + + // There's no such case. + //char t161[] = "' A BC '"; + //EXPECT_EQ(testValidateName(t161), TSDB_CODE_INVALID_SQL); + + char t162[] = " AB C . de f "; + EXPECT_EQ(testValidateName(t162), TSDB_CODE_INVALID_SQL); + + char t163[] = "' AB C . de f "; + EXPECT_EQ(testValidateName(t163), TSDB_CODE_INVALID_SQL); + + char t163_0[] = " ab c . DE F ' "; + EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_INVALID_SQL); + + char t164[] = " ' AB C . de f ' "; + //int32_t tmp2 = testValidateName(t164); + EXPECT_EQ(testValidateName(t164), TSDB_CODE_INVALID_SQL); + + char t165[] = " ' A BC '. de f "; + EXPECT_EQ(testValidateName(t165), TSDB_CODE_INVALID_SQL); + + char t166[] = "' AB C '.' DE F '"; + EXPECT_EQ(testValidateName(t166), TSDB_CODE_INVALID_SQL); + + char t167[] = "ab c . ' D EF '"; + EXPECT_EQ(testValidateName(t167), TSDB_CODE_INVALID_SQL); + + char t168[] = "' a bc '.' DE F '"; + EXPECT_EQ(testValidateName(t168), TSDB_CODE_INVALID_SQL); + +} + +/* test parse time function */ +TEST(testCase, parse_time) { + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + char t1[] = "2018-1-1 1:1:1.952798"; + char t13[] = "1970-1-1 0:0:0"; + + int64_t time = 0, time1 = 0; + + taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); + + char t2[] = "2018-1-1T1:1:1.952Z"; + taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI); + + EXPECT_EQ(time, 1514739661952 + 28800000); + + char t3[] = "2018-1-1 1:01:01.952"; + taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + char t4[] = "2018-1-1 1:01:01.9"; + char t5[] = "2018-1-1 1:01:1.900"; + char t6[] = "2018-01-01 1:1:1.90"; + char t7[] = "2018-01-01 01:01:01.9"; + char t8[] = "2018-01-01 01:01:01.9007865"; + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t9[] = "2017-4-3 1:1:2.980"; + char t10[] = "2017-4-3T2:1:2.98+9:00"; + taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t11[] = "2017-4-3T2:1:2.98+09:00"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t12[] = "2017-4-3T2:1:2.98+0900"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taos_options(TSDB_OPTION_TIMEZONE, "UTC"); + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + char t14[] = "1970-1-1T0:0:0Z"; + taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + char t40[] = "1970-1-1 0:0:0.999999999"; + taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); + + char t41[] = "1997-1-1 0:0:0.999999999"; + taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999); + + int64_t k = timezone; + char t42[] = "1997-1-1T0:0:0.999999999Z"; + taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); + + //////////////////////////////////////////////////////////////////// + // illegal timestamp format + char t15[] = "2017-12-33 0:0:0"; + EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1); + + char t16[] = "2017-12-31 99:0:0"; + EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1); + + char t17[] = "2017-12-31T9:0:0"; + EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1); + + char t18[] = "2017-12-31T9:0:0.Z"; + EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1); + + char t19[] = "2017-12-31 9:0:0.-1"; + EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1); + + char t20[] = "2017-12-31 9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0); + EXPECT_EQ(time, 1514682000100); + + char t21[] = "2017-12-31T9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1); + + char t22[] = "2017-12-31 9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0); + + char t23[] = "2017-12-31T9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0); +} + +TEST(testCase, tvariant_convert) { + // 1. bool data to all other data types + tVariant t = {0}; + _init_tvariant_bool(&t); + + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "FALSE"); + tVariantDestroy(&t); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"FALSE"); + tVariantDestroy(&t); + + // 2. tinyint to other data types + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-27"); + tVariantDestroy(&t); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-27"); + tVariantDestroy(&t); + + // 3. int to other data + // types////////////////////////////////////////////////////////////////// + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + EXPECT_EQ(t.i64Key, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-23997659"); + tVariantDestroy(&t); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-23997659"); + tVariantDestroy(&t); + + // 4. bigint to other data + // type////////////////////////////////////////////////////////////////////////////// + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-3333333333333"); + tVariantDestroy(&t); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-3333333333333"); + tVariantDestroy(&t); + + // 5. float to other data + // types//////////////////////////////////////////////////////////////////////// + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -8991212199); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-8991212199.898788"); + tVariantDestroy(&t); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-8991212199.898788"); + tVariantDestroy(&t); + + // 6. binary to other data types + // ////////////////////////////////////////////////////////////////// + t.pz = "true"; + t.nLen = strlen(t.pz); + t.nType = TSDB_DATA_TYPE_BINARY; + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "2e5"); + tVariantDestroy(&t); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"2e5"); + tVariantDestroy(&t); + + // 7. nchar to other data types + // ////////////////////////////////////////////////////////////////// + t.wpz = L"FALSE"; + t.nLen = wcslen(t.wpz); + t.nType = TSDB_DATA_TYPE_NCHAR; + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_nchar(&t); + EXPECT_LE(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -2000000); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-2000000.8765"); + tVariantDestroy(&t); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-2000000.8765"); + tVariantDestroy(&t); +} + +TEST(testCase, tGetToken_Test) { + char* s = ".123 "; + uint32_t type = 0; + + int32_t len = tSQLGetToken(s, &type); + EXPECT_EQ(type, TK_FLOAT); + EXPECT_EQ(len, strlen(s) - 1); + + char s1[] = "1.123e10 "; + len = tSQLGetToken(s1, &type); + EXPECT_EQ(type, TK_FLOAT); + EXPECT_EQ(len, strlen(s1) - 1); + + char s4[] = "0xff "; + len = tSQLGetToken(s4, &type); + EXPECT_EQ(type, TK_HEX); + EXPECT_EQ(len, strlen(s4) - 1); + + // invalid data type + char s2[] = "e10 "; + len = tSQLGetToken(s2, &type); + EXPECT_FALSE(type == TK_FLOAT); + + char s3[] = "1.1.1.1"; + len = tSQLGetToken(s3, &type); + EXPECT_EQ(type, TK_IPTOKEN); + EXPECT_EQ(len, strlen(s3)); + + char s5[] = "0x "; + len = tSQLGetToken(s5, &type); + EXPECT_FALSE(type == TK_HEX); +} + +static SSQLToken createStrToken(char* s) { + SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; + t.type = TK_STRING; + t.z = s; + t.n = strlen(s); + + return t; +} + +TEST(testCase, isValidNumber_test) { + SSQLToken t1 = createStrToken("123abc"); + + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("0xabc"); + EXPECT_EQ(isValidNumber(&t1), TK_HEX); + + t1 = createStrToken("0b11101"); + EXPECT_EQ(isValidNumber(&t1), TK_BIN); + + t1 = createStrToken(".134abc"); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("1e1 "); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("1+2"); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("-0x123"); + EXPECT_EQ(isValidNumber(&t1), TK_HEX); + + t1 = createStrToken("-1"); + EXPECT_EQ(isValidNumber(&t1), TK_INTEGER); + + t1 = createStrToken("-0b1110"); + EXPECT_EQ(isValidNumber(&t1), TK_BIN); + + t1 = createStrToken("-.234"); + EXPECT_EQ(isValidNumber(&t1), TK_FLOAT); +} + +TEST(testCase, getTempFilePath_test) { + char path[4096] = {0}; + memset(path, 1, 4096); + + getTmpfilePath("new_tmp", path); + printf("%s\n", path); +} + diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index e0d219188846ba47ec370a66ca388b2685a4df32..21ce7ee60b951a5bed391ef68e204c5508fb824e 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -32,6 +32,7 @@ #include "rpcServer.h" #include "rpcHead.h" #include "trpc.h" +#include "hash.h" #define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) #define rpcHeadFromCont(cont) ((SRpcHead *) (cont - sizeof(SRpcHead))) @@ -262,7 +263,8 @@ void *rpcOpen(SRpcInit *pInit) { } if (pRpc->connType == TAOS_CONN_SERVER) { - pRpc->hash = taosInitStrHash(pRpc->sessions, sizeof(pRpc), taosHashString); +// pRpc->hash = taosInitStrHash(pRpc->sessions, sizeof(pRpc), taosHashString); + pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true); if (pRpc->hash == NULL) { tError("%s failed to init string hash", pRpc->label); rpcClose(pRpc); @@ -296,7 +298,8 @@ void rpcClose(void *param) { } } - taosCleanUpStrHash(pRpc->hash); +// taosCleanUpStrHash(pRpc->hash); + taosHashCleanup(pRpc->hash); taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); rpcCloseConnCache(pRpc->pCache); @@ -364,7 +367,7 @@ void rpcSendRequest(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg) { // for TDengine, all the query, show commands shall have TCP connection char type = pMsg->msgType; if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_RETRIEVE || - type == TSDB_MSG_TYPE_CM_STABLE_META || type == TSDB_MSG_TYPE_CM_TABLES_META || + type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_SHOW ) pContext->connType = RPC_CONN_TCPC; @@ -531,8 +534,10 @@ static void rpcCloseConn(void *thandle) { if ( pRpc->connType == TAOS_CONN_SERVER) { char hashstr[40] = {0}; - sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); - taosDeleteStrHash(pRpc->hash, hashstr); + size_t size = sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); +// taosDeleteStrHash(pRpc->hash, hashstr); +// taosHashRemove(pRpc->hash, hashstr, size); + rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg pConn->pRspMsg = NULL; pConn->inType = 0; @@ -580,10 +585,11 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { char hashstr[40] = {0}; SRpcHead *pHead = (SRpcHead *)pRecv->msg; - sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); + size_t size = sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); // check if it is already allocated - SRpcConn **ppConn = (SRpcConn **)(taosGetStrHashData(pRpc->hash, hashstr)); +// SRpcConn **ppConn = (SRpcConn **)(taosGetStrHashData(pRpc->hash, hashstr)); + SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); if (ppConn) pConn = *ppConn; if (pConn) return pConn; @@ -615,7 +621,9 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { pConn->localPort = (pRpc->localPort + pRpc->index); } - taosAddStrHash(pRpc->hash, hashstr, (char *)&pConn); +// taosAddStrHash(pRpc->hash, hashstr, (char *)&pConn); + taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); + tTrace("%s %p, rpc connection is allocated, sid:%d id:%s port:%u", pRpc->label, pConn, sid, pConn->user, pConn->localPort); } @@ -863,18 +871,19 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->port); } - if (terrno != TSDB_CODE_ALREADY_PROCESSED) { - if (terrno != 0) { // parsing error + int32_t code = terrno; + if (code != TSDB_CODE_ALREADY_PROCESSED) { + if (code != 0) { // parsing error if ( rpcIsReq(pHead->msgType) ) { - rpcSendErrorMsgToPeer(pRecv, terrno); - tTrace("%s %p, %s is sent with error code:0x%x", pRpc->label, pConn, taosMsg[pHead->msgType+1], terrno); + rpcSendErrorMsgToPeer(pRecv, code); + tTrace("%s %p, %s is sent with error code:%x", pRpc->label, pConn, taosMsg[pHead->msgType+1], code); } } else { // parsing OK rpcProcessIncomingMsg(pConn, pHead); } } - if (terrno) rpcFreeMsg(pRecv->msg); + if (code) rpcFreeMsg(pRecv->msg); return pConn; } diff --git a/src/sdb/CMakeLists.txt b/src/sdb/CMakeLists.txt deleted file mode 100644 index 47ea6e15b8cf5d8ba6f5d24c562e2b133f58f5d0..0000000000000000000000000000000000000000 --- a/src/sdb/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) -INCLUDE_DIRECTORIES(inc) - -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) - AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(sdb ${SRC}) - TARGET_LINK_LIBRARIES(sdb trpc) - IF (TD_CLUSTER) - TARGET_LINK_LIBRARIES(sdb) - ENDIF() -ENDIF () diff --git a/src/sdb/inc/sdbint.h b/src/sdb/inc/sdbint.h deleted file mode 100644 index 30ebe0909c83f5fdefd6105e31f0ecd467d1b77d..0000000000000000000000000000000000000000 --- a/src/sdb/inc/sdbint.h +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _sdbint_header_ -#define _sdbint_header_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "hashint.h" -#include "hashstr.h" -#include "sdb.h" -#include "tchecksum.h" -#include "tlog.h" -#include "trpc.h" -#include "tutil.h" - -#define sdbError(...) \ - if (sdbDebugFlag & DEBUG_ERROR) { \ - tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \ - } -#define sdbWarn(...) \ - if (sdbDebugFlag & DEBUG_WARN) { \ - tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ - } -#define sdbTrace(...) \ - if (sdbDebugFlag & DEBUG_TRACE) { \ - tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ - } -#define sdbPrint(...) \ - { tprintf("MND-SDB ", 255, __VA_ARGS__); } - -#define mpeerError(...) \ - if (sdbDebugFlag & DEBUG_ERROR) { \ - tprintf("ERROR MND-MPEER ", 255, __VA_ARGS__); \ - } -#define mpeerWarn(...) \ - if (sdbDebugFlag & DEBUG_WARN) { \ - tprintf("WARN MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \ - } -#define mpeerTrace(...) \ - if (sdbDebugFlag & DEBUG_TRACE) { \ - tprintf("MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \ - } -#define mpeerPrint(...) \ - { tprintf("MND-MPEER ", 255, __VA_ARGS__); } - -#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__) -#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__) -#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__) - -#define SDB_MAX_PEERS 4 -#define SDB_DELIMITER 0xFFF00F00 -#define SDB_ENDCOMMIT 0xAFFFAAAF - -typedef struct { - uint64_t swVersion; - int16_t sdbFileVersion; - char reserved[6]; - TSCKSUM checkSum; -} SSdbHeader; - -typedef struct { - char type; - // short rowSize; - char *row; -} SSdbUpdate; - -typedef struct _SSdbTable { - SSdbHeader header; - int maxRows; - int dbId; - int32_t maxRowSize; - char name[TSDB_DB_NAME_LEN]; - char fn[128]; - int keyType; - uint32_t autoIndex; - int64_t numOfRows; - int64_t id; - int64_t size; - void * iHandle; - int fd; - void *(*appTool)(char, void *, char *, int, int *); - pthread_mutex_t mutex; - SSdbUpdate * update; - int numOfUpdates; - int updatePos; -} SSdbTable; - -typedef struct { - int64_t id; - int64_t offset; - int rowSize; - void * row; -} SRowMeta; - -typedef struct { - int32_t delimiter; - int32_t rowSize; - int64_t id; - char data[]; -} SRowHead; - -typedef struct { - uint8_t dbId; - char type; - uint64_t version; - short dataLen; - char data[]; -} SForwardMsg; - -extern SSdbTable *tableList[]; -extern int sdbMaxPeers; -extern int sdbNumOfTables; -extern int64_t sdbVersion; - -int sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int dataLen); -int mpeerRetrieveRows(int fd, SSdbTable *pTable, uint64_t version); -void sdbResetTable(SSdbTable *pTable); -extern const int16_t sdbFileVersion; - - -#endif diff --git a/src/sdb/src/sdbEngine.c b/src/sdb/src/sdbEngine.c deleted file mode 100644 index fbc41089d1c9917aa5adae26b0052450a44e844d..0000000000000000000000000000000000000000 --- a/src/sdb/src/sdbEngine.c +++ /dev/null @@ -1,1017 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" - -#include "sdb.h" -#include "sdbint.h" -#include "tutil.h" - -#define abs(x) (((x) < 0) ? -(x) : (x)) - -extern char version[]; -const int16_t sdbFileVersion = 0; -SRpcIpSet *pSdbIpList = NULL; -SRpcIpSet *pSdbPublicIpList = NULL; -SSdbPeer * sdbPeer[SDB_MAX_PEERS]; // first slot for self - -#ifdef CLUSTER -int sdbMaster = 0; -#else -int sdbMaster = 1; -#endif - -void *(*sdbInitIndexFp[])(int maxRows, int dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash}; - -void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash}; - -void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash}; - -void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData}; - -void (*sdbCleanUpIndexFp[])(void *handle) = { - sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash, -}; - -void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = { - sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData, -}; - -SSdbTable *tableList[20]; -int sdbNumOfTables; -int64_t sdbVersion; - -int64_t sdbGetVersion() { - return sdbVersion; -}; - -int32_t sdbGetRunStatus() { - if (!tsIsCluster) { - return SDB_STATUS_SERVING; - } - - if (sdbInited == NULL) { - return SDB_STATUS_OFFLINE; - } - return sdbStatus; -} - -void sdbFinishCommit(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - uint32_t sdbEcommit = SDB_ENDCOMMIT; - - off_t offset = lseek(pTable->fd, 0, SEEK_END); - assert(offset == pTable->size); - twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - pTable->size += sizeof(sdbEcommit); -} - -int sdbOpenSdbFile(SSdbTable *pTable) { - struct stat fstat, ofstat; - uint64_t size; - char * dirc = NULL; - char * basec = NULL; - union { - char cversion[64]; - uint64_t iversion; - } swVersion; - - memcpy(swVersion.cversion, version, sizeof(uint64_t)); - - // check sdb.db and .sdb.db status - char fn[128] = "\0"; - dirc = strdup(pTable->fn); - basec = strdup(pTable->fn); - sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - tfree(dirc); - tfree(basec); - if (stat(fn, &ofstat) == 0) { // .sdb.db file exists - if (stat(pTable->fn, &fstat) == 0) { - remove(fn); - } else { - remove(pTable->fn); - rename(fn, pTable->fn); - } - } - - pTable->fd = open(pTable->fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - if (pTable->fd < 0) { - sdbError("failed to open file:%s", pTable->fn); - return -1; - } - - pTable->size = 0; - stat(pTable->fn, &fstat); - size = sizeof(pTable->header); - - if (fstat.st_size == 0) { - pTable->header.swVersion = swVersion.iversion; - pTable->header.sdbFileVersion = sdbFileVersion; - if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) { - sdbError("failed to get file header checksum, file:%s", pTable->fn); - tclose(pTable->fd); - return -1; - } - twrite(pTable->fd, &(pTable->header), size); - pTable->size += size; - sdbFinishCommit(pTable); - } else { - uint32_t sdbEcommit = 0; - off_t offset = lseek(pTable->fd, -(sizeof(sdbEcommit)), SEEK_END); - while (offset > 0) { - read(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - if (sdbEcommit == SDB_ENDCOMMIT) { - ftruncate(pTable->fd, offset + sizeof(sdbEcommit)); - break; - } - offset = lseek(pTable->fd, -(sizeof(sdbEcommit) + 1), SEEK_CUR); - } - lseek(pTable->fd, 0, SEEK_SET); - - ssize_t tsize = read(pTable->fd, &(pTable->header), size); - if (tsize < size) { - sdbError("failed to read sdb file header, file:%s", pTable->fn); - tclose(pTable->fd); - return -1; - } - - if (pTable->header.swVersion != swVersion.iversion) { - sdbWarn("sdb file:%s version not match software version", pTable->fn); - } - - if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) { - sdbError("sdb file header is broken since checksum mismatch, file:%s", pTable->fn); - tclose(pTable->fd); - return -1; - } - - pTable->size += size; - // skip end commit symbol - lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR); - pTable->size += sizeof(sdbEcommit); - } - - pTable->numOfRows = 0; - - return pTable->fd; -} - -// TODO: Change here -void sdbAddIntoUpdateList(SSdbTable *pTable, char type, char *row) { - pTable->numOfUpdates++; - pTable->updatePos = pTable->numOfUpdates % pTable->maxRows; - - if (pTable->update[pTable->updatePos].type == SDB_TYPE_DELETE) - (*(pTable->appTool))(SDB_TYPE_DESTROY, pTable->update[pTable->updatePos].row, NULL, 0, NULL); - - pTable->update[pTable->updatePos].type = type; - pTable->update[pTable->updatePos].row = row; -} - -int sdbInitTableByFile(SSdbTable *pTable) { - SRowMeta rowMeta; - int numOfDels = 0; - int bytes = 0; - int64_t oldId = 0; - void * pMetaRow = NULL; - int total_size = 0; - int real_size = 0; - int maxAutoIndex = 0; - - oldId = pTable->id; - if (sdbOpenSdbFile(pTable) < 0) return -1; - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - - sdbTrace("open sdb file:%s for read", pTable->fn); - - // Loop to read sdb file row by row - while (1) { - memset(rowHead, 0, total_size); - - bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); - if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fn); - goto sdb_exit1; - } - - if (bytes == 0) break; - - if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->size++; - lseek(pTable->fd, -(bytes - 1), SEEK_CUR); - continue; - } - - if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s, id:%d rowSize:%d maxRowSize:%d", - pTable->fn, rowHead->id, rowHead->rowSize, pTable->maxRowSize); - pTable->size += sizeof(SRowHead); - continue; - } - - // sdbTrace("%s id:%ld rowSize:%d", pTable->name, rowHead->id, - // rowHead->rowSize); - - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); - if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - // TODO: Here may cause pTable->size not end of the file - sdbError("failed to read sdb file:%s id:%d rowSize:%d", pTable->fn, rowHead->id, rowHead->rowSize); - break; - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s id:%d, skip", pTable->name, rowHead->id); - pTable->size += real_size; - continue; - } - - // Check if the the object exists already - - pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { // New object - if (rowHead->id < 0) { - /* assert(0); */ - sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); - } else { - rowMeta.id = rowHead->id; - // TODO: Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex++; - maxAutoIndex = MAX(maxAutoIndex, *(int32_t*)rowHead->data); - } - pTable->numOfRows++; - } - } else { // already exists - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex++; - maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); - } - - if (rowHead->id < 0) { // Delete the object - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL); - pTable->numOfRows--; - numOfDels++; - } else { // Reset the object TODO: is it possible to merge reset and - // update ?? - (*(pTable->appTool))(SDB_TYPE_RESET, pMetaRow, rowHead->data, rowHead->rowSize, NULL); - } - numOfDels++; - } - - pTable->size += real_size; - if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); - } - - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex = maxAutoIndex; - } - - sdbVersion += (pTable->id - oldId); - if (numOfDels > pTable->maxRows / 4) sdbSaveSnapShot(pTable); - - pTable->numOfUpdates = 0; - pTable->updatePos = 0; - - tfree(rowHead); - return 0; - -sdb_exit1: - tfree(rowHead); - return -1; -} - -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int, int *)) { - SSdbTable *pTable = (SSdbTable *)malloc(sizeof(SSdbTable)); - if (pTable == NULL) return NULL; - memset(pTable, 0, sizeof(SSdbTable)); - - int size = sizeof(SSdbUpdate) * maxRows; - pTable->update = (SSdbUpdate *)malloc(size); - if (pTable->update == NULL) { - free(pTable); - return NULL; - }; - memset(pTable->update, 0, size); - - strcpy(pTable->name, name); - pTable->keyType = keyType; - pTable->maxRows = maxRows; - pTable->maxRowSize = maxRowSize; - pTable->appTool = appTool; - sprintf(pTable->fn, "%s/%s.db", directory, pTable->name); - - if (sdbInitIndexFp[keyType] != NULL) pTable->iHandle = (*sdbInitIndexFp[keyType])(maxRows, sizeof(SRowMeta)); - - pthread_mutex_init(&pTable->mutex, NULL); - - if (sdbInitTableByFile(pTable) < 0) return NULL; - - pTable->dbId = sdbNumOfTables++; - tableList[pTable->dbId] = pTable; - - sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d", pTable->name, pTable->numOfRows, sdbNumOfTables); - - return pTable; -} - -SRowMeta *sdbGetRowMeta(void *handle, void *key) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - - if (handle == NULL) return NULL; - - pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); - - return pMeta; -} - -void *sdbGetRow(void *handle, void *key) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - - if (handle == NULL) return NULL; - - pthread_mutex_lock(&pTable->mutex); - pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); - pthread_mutex_unlock(&pTable->mutex); - - if (pMeta == NULL) return NULL; - - return pMeta->row; -} - -// row here must be encoded string (rowSize > 0) or the object it self (rowSize -// = 0) -int64_t sdbInsertRow(void *handle, void *row, int rowSize) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta rowMeta; - int64_t id = -1; - void * pObj = NULL; - int total_size = 0; - int real_size = 0; - /* char action = SDB_TYPE_INSERT; */ - - if (pTable == NULL) { - sdbError("sdb tables is null"); - return -1; - } - - if ((pTable->keyType != SDB_KEYTYPE_AUTO) || *((int64_t *)row)) - if (sdbGetRow(handle, row)) { - if (strcmp(pTable->name, "mnode") == 0) { - /* - * The first mnode created when the system just start, so the insert action may failed - * see sdbPeer.c : sdbInitPeers - */ - pTable->id++; - sdbVersion++; - sdbPrint("table:%s, record:%s already exist, think it successed, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id); - return 0; - } else { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to insert record sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); - break; - } - return -1; - } - } - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb: %s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - - if (rowSize == 0) { // object is created already - pObj = row; - } else { // encoded string, to create object - pObj = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, row, rowSize, NULL); - } - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); - - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_INSERT, rowHead->data, rowHead->rowSize) == 0) { - pTable->id++; - sdbVersion++; - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - // TODO:here need to change - *((uint32_t *)pObj) = ++pTable->autoIndex; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - // update in SDB layer - rowMeta.id = pTable->id; - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); - - /* Update the disk content */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, real_size); - pTable->size += real_size; - sdbFinishCommit(pTable); - - sdbAddIntoUpdateList(pTable, SDB_TYPE_INSERT, rowMeta.row); - - pTable->numOfRows++; - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - default: - sdbTrace("table:%s, a record is inserted, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - } - - id = rowMeta.id; - } else { - sdbError("table:%s, failed to insert record", pTable->name); - } - - tfree(rowHead); - - pthread_mutex_unlock(&pTable->mutex); - - /* callback function to update the MGMT layer */ - if (id >= 0 && pTable->appTool) (*pTable->appTool)(SDB_TYPE_INSERT, pObj, NULL, 0, NULL); - - return id; -} - -// row here can be object or null-terminated string -int sdbDeleteRow(void *handle, void *row) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int code = -1; - void * pMetaRow = NULL; - SRowHead * rowHead = NULL; - int rowSize = 0; - int total_size = 0; - /* char action = SDB_TYPE_DELETE; */ - - if (pTable == NULL) return -1; - - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, delete failed", pTable->name); - return -1; - } - - pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - rowSize = strlen((char *)row) + 1; - break; - case SDB_KEYTYPE_UINT32: - rowSize = sizeof(uint32_t); - break; - case SDB_KEYTYPE_AUTO: - rowSize = sizeof(uint64_t); - break; - default: - return -1; - } - - total_size = sizeof(SRowHead) + rowSize + sizeof(TSCKSUM); - rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_DELETE, (char *)row, rowSize) == 0) { - pTable->id++; - sdbVersion++; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->id = -(pTable->id); - memcpy(rowHead->data, row, rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, total_size); - pTable->size += total_size; - sdbFinishCommit(pTable); - - pTable->numOfRows--; - // TODO:Change the update list here - sdbAddIntoUpdateList(pTable, SDB_TYPE_DELETE, pMetaRow); - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is deleted, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, sdbVersion, pTable->id, pTable->numOfRows); - break; - } - - // Delete from current layer - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); - - code = 0; - } - - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - // callback function of the delete - if (code == 0 && pTable->appTool) (*pTable->appTool)(SDB_TYPE_DELETE, pMetaRow, NULL, 0, NULL); - - return code; -} - -// row here can be the object or the string info (encoded string) -int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int code = -1; - int total_size = 0; - int real_size = 0; - /* char action = SDB_TYPE_UPDATE; */ - - if (pTable == NULL || row == NULL) return -1; - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, (char *) row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, taosIpStr(*(int32_t *) row), sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to update record:%d, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, *(int32_t *) row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to update record, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, sdbVersion, pTable->id); - break; - } - return -1; - } - - void *pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - - if (!isUpdated) { - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, row, updateSize, NULL); // update in upper layer - } - - if (pMetaRow != row) { - memcpy(rowHead->data, row, updateSize); - rowHead->rowSize = updateSize; - } else { - (*(pTable->appTool))(SDB_TYPE_ENCODE, pMetaRow, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - ; - - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_UPDATE, rowHead->data, rowHead->rowSize) == 0) { - pTable->id++; - sdbVersion++; - - // write to the new position - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, real_size); - - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - pMeta->rowSize = rowHead->rowSize; - pTable->size += real_size; - - sdbFinishCommit(pTable); - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is updated, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, sdbVersion, - pTable->id, pTable->numOfRows); - break; - } - - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, pMetaRow); - code = 0; - } - - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - return code; -} - -// row here must be the instruction string -int sdbBatchUpdateRow(void *handle, void *row, int rowSize) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int total_size = 0; - /* char action = SDB_TYPE_BATCH_UPDATE; */ - - if (pTable == NULL || row == NULL || rowSize <= 0) return -1; - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, batch update failed", pTable->name); - return -1; - } - - void *pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - - pthread_mutex_lock(&pTable->mutex); - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_BATCH_UPDATE, row, rowSize) == 0) { - /* // write action */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - - (*(pTable->appTool))(SDB_TYPE_BEFORE_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - - void *next_row = pMetaRow; - while (next_row != NULL) { - pTable->id++; - sdbVersion++; - - void *last_row = next_row; - next_row = (*(pTable->appTool))(SDB_TYPE_BATCH_UPDATE, last_row, (char *)row, rowSize, 0); - memset(rowHead, 0, sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM)); - - // update in current layer - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - - // write to disk - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, last_row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - taosCalcChecksumAppend(0, (uint8_t *)rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pMeta->rowSize = rowHead->rowSize; - lseek(pTable->fd, pTable->size, SEEK_SET); - twrite(pTable->fd, rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pTable->size += (sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, last_row); - - if (next_row != NULL) { - pMeta = sdbGetRowMeta(handle, next_row); - } - } - - sdbFinishCommit(pTable); - - (*(pTable->appTool))(SDB_TYPE_AFTER_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - } - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - return 0; -} - -void sdbCloseTable(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - void * pNode = NULL; - void * row; - - if (pTable == NULL) return; - - while (1) { - pNode = sdbFetchRow(handle, pNode, &row); - if (row == NULL) break; - (*(pTable->appTool))(SDB_TYPE_DESTROY, row, NULL, 0, NULL); - } - - if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); - - if (pTable->fd) tclose(pTable->fd); - - pthread_mutex_destroy(&pTable->mutex); - - sdbNumOfTables--; - sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->name, pTable->id, sdbNumOfTables); - - tfree(pTable->update); - tfree(pTable); -} - -void sdbResetTable(SSdbTable *pTable) { - /* SRowHead rowHead; */ - SRowMeta rowMeta; - int bytes; - int total_size = 0; - int real_size = 0; - SRowHead *rowHead = NULL; - void * pMetaRow = NULL; - int64_t oldId = pTable->id; - int oldNumOfRows = pTable->numOfRows; - - if (sdbOpenSdbFile(pTable) < 0) return; - pTable->numOfRows = oldNumOfRows; - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory for reset, sdb:%s", pTable->name); - return; - } - - sdbPrint("open sdb file:%s for reset table", pTable->fn); - - while (1) { - memset(rowHead, 0, total_size); - - bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); - if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fn); - tfree(rowHead); - return; - } - - if (bytes == 0) break; - - if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->size++; - lseek(pTable->fd, -(bytes - 1), SEEK_CUR); - continue; - } - - if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s for reset, id:%d rowSize:%d maxRowSize:%d", - pTable->fn, rowHead->id, rowHead->rowSize, pTable->maxRowSize); - pTable->size += sizeof(SRowHead); - continue; - } - - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); - if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - sdbError("failed to read sdb file:%s for reset, id:%d rowSize:%d", pTable->fn, rowHead->id, rowHead->rowSize); - break; - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s id:%d, skip", pTable->name, rowHead->id); - pTable->size += real_size; - continue; - } - - if (abs(rowHead->id) > oldId) { // not operated - pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { // New object - if (rowHead->id < 0) { - sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); - } else { - rowMeta.id = rowHead->id; - // TODO:Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - pTable->numOfRows++; - - (*pTable->appTool)(SDB_TYPE_INSERT, rowMeta.row, NULL, 0, NULL); - } - } else { // already exists - if (rowHead->id < 0) { // Delete the object - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL); - pTable->numOfRows--; - } else { // update the object - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, rowHead->data, rowHead->rowSize, NULL); - } - } - } - - pTable->size += real_size; - if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); - } - - sdbVersion += (pTable->id - oldId); - pTable->numOfUpdates = 0; - pTable->updatePos = 0; - - tfree(rowHead); - - sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); -} - -// TODO:A problem here :use snapshot file to sync another node will cause -// problem -void sdbSaveSnapShot(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - void * pNode = NULL; - int total_size = 0; - int real_size = 0; - int size = 0; - int numOfRows = 0; - uint32_t sdbEcommit = SDB_ENDCOMMIT; - char * dirc = NULL; - char * basec = NULL; - /* char action = SDB_TYPE_INSERT; */ - - if (pTable == NULL) return; - - sdbTrace("Table:%s, save the snapshop", pTable->name); - - char fn[128] = "\0"; - dirc = strdup(pTable->fn); - basec = strdup(pTable->fn); - sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - int fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - tfree(dirc); - tfree(basec); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate memory while saving SDB snapshot, sdb:%s", pTable->name); - return; - } - memset(rowHead, 0, size); - - // Write the header - twrite(fd, &(pTable->header), sizeof(SSdbHeader)); - size += sizeof(SSdbHeader); - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - - while (1) { - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) break; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pMeta->row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while save sdb %s snapshot", pTable->name); - tfree(rowHead); - return; - } - - /* write(fd, &action, sizeof(action)); */ - /* size += sizeof(action); */ - twrite(fd, rowHead, real_size); - size += real_size; - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - numOfRows++; - } - - tfree(rowHead); - - // Remove the old file - tclose(pTable->fd); - remove(pTable->fn); - // Rename the .sdb.db file to sdb.db file - rename(fn, pTable->fn); - pTable->fd = fd; - pTable->size = size; - pTable->numOfRows = numOfRows; - - fdatasync(pTable->fd); -} - -void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - - *ppRow = NULL; - if (pTable == NULL) return NULL; - - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) return NULL; - - *ppRow = pMeta->row; - - return pNode; -} - -int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } - -int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } diff --git a/src/sdb/src/sdbstr.c b/src/sdb/src/sdbstr.c deleted file mode 100644 index 59c01eb15a95a7354b5f37acb1d521a5e89c100d..0000000000000000000000000000000000000000 --- a/src/sdb/src/sdbstr.c +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#define _DEFAULT_SOURCE -#include "sdbint.h" - -int32_t (*mpeerInitMnodesFp)(char *directory) = NULL; -void (*mpeerCleanUpMnodesFp)() = NULL; -int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; - -char *sdbStatusStr[] = { - "offline", - "unsynced", - "syncing", - "serving", - "null" -}; - -char *sdbRoleStr[] = { - "unauthed", - "undecided", - "master", - "slave", - "null" -}; - -int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { - if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(pTable, type, data, dataLen); - } else { - return 0; - } -} - -int32_t sdbInitPeers(char *directory) { - if (mpeerInitMnodesFp) { - return (*mpeerInitMnodesFp)(directory); - } else { - return 0; - } -} - -void sdbCleanUpPeers() { - if (mpeerCleanUpMnodesFp) { - (*mpeerCleanUpMnodesFp)(); - } -} diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 67ecf39f1b2a0564e8f2bcdc0cb39360652a3965..0844146bd4667a58666f4807c98ce8a19884a4bd 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -29,6 +29,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) MESSAGE(STATUS "Failed to find iconv, use default encoding method") ENDIF () ENDIF () + + ADD_SUBDIRECTORY(tests) ELSEIF (TD_WINDOWS_64) ADD_DEFINITIONS(-DUSE_LIBICONV) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) @@ -113,4 +115,3 @@ ENDIF() #ENDIF () - diff --git a/src/sdb/inc/hashint.h b/src/util/inc/hashint.h similarity index 100% rename from src/sdb/inc/hashint.h rename to src/util/inc/hashint.h diff --git a/src/sdb/inc/hashstr.h b/src/util/inc/hashstr.h similarity index 100% rename from src/sdb/inc/hashstr.h rename to src/util/inc/hashstr.h diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 55bdc849adf4bf6d8941f61c669f73cdcf2880fd..6b70780d10482610ef64b1c50640756db4204bd6 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -87,7 +87,21 @@ size_t taosArrayGetSize(const SArray* pArray); void* taosArrayInsert(SArray* pArray, size_t index, void* pData); /** - * + * remove data entry of the given index + * @param pArray + * @param index + */ +void taosArrayRemove(SArray* pArray, size_t index); + +/** + * copy the whole array from source to destination + * @param pDst + * @param pSrc + */ +void taosArrayCopy(SArray* pDst, SArray* pSrc); + +/** + * destroy array list * @param pArray */ void taosArrayDestroy(SArray* pArray); diff --git a/src/util/inc/tglobalcfg.h b/src/util/inc/tglobalcfg.h index 993992ffcbf9a33b00761dd20141224f912db958..664eb163734eef0eccdb65a315ee89a816b9f530 100644 --- a/src/util/inc/tglobalcfg.h +++ b/src/util/inc/tglobalcfg.h @@ -103,13 +103,7 @@ extern int tsReplications; extern int tsNumOfMPeers; extern int tsMaxShellConns; -extern int tsMaxAccounts; -extern int tsMaxUsers; -extern int tsMaxDbs; extern int tsMaxTables; -extern int tsMaxDnodes; -extern int tsMaxVGroups; -extern char tsMgmtZone[]; extern char tsLocalIp[]; extern char tsDefaultDB[]; diff --git a/src/util/inc/tidpool.h b/src/util/inc/tidpool.h index 08f9d364671a5d126e699c26cb92e74365be0c78..bf352516310a1356f37359ab3052814d726ab8f0 100644 --- a/src/util/inc/tidpool.h +++ b/src/util/inc/tidpool.h @@ -34,11 +34,7 @@ void taosIdPoolCleanUp(void *handle); int taosIdPoolNumOfUsed(void *handle); -void taosIdPoolReinit(void *handle); - -void taosIdPoolMarkStatus(void *handle, int id, int status); - -void taosIdPoolSetFreeList(void *handle); +void taosIdPoolMarkStatus(void *handle, int id); #ifdef __cplusplus } diff --git a/src/util/inc/tlog.h b/src/util/inc/tlog.h index e1c5f8a06a5d9476bc53488e91a71693b295e931..8a3bd6bc7c9b2728430f5341d0f1b0c94bc9b17c 100644 --- a/src/util/inc/tlog.h +++ b/src/util/inc/tlog.h @@ -239,6 +239,25 @@ extern uint32_t cdebugFlag; #define monitorLWarn(...) taosLogWarn(__VA_ARGS__) monitorWarn(__VA_ARGS__) #define monitorLPrint(...) taosLogPrint(__VA_ARGS__) monitorPrint(__VA_ARGS__) +#define sdbError(...) \ + if (sdbDebugFlag & DEBUG_ERROR) { \ + tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \ + } +#define sdbWarn(...) \ + if (sdbDebugFlag & DEBUG_WARN) { \ + tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ + } +#define sdbTrace(...) \ + if (sdbDebugFlag & DEBUG_TRACE) { \ + tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ + } +#define sdbPrint(...) \ + { tprintf("MND-SDB ", 255, __VA_ARGS__); } + +#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__) +#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__) +#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__) + #ifdef __cplusplus } #endif diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 0ec1d0eab52298913029f455e1566d9cf5d6630b..18404f89c27d2c122e813edae674bb3869c451b7 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -51,7 +51,7 @@ typedef struct SSkipListNode { #define SL_GET_BACKWARD_POINTER(n, _l) \ ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)] -#define SL_GET_NODE_DATA(n) ((char*)(n) + SL_NODE_HEADER_SIZE((n)->level)) +#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level)) #define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n))) #define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n)) @@ -106,25 +106,25 @@ typedef struct tSkipListState { typedef struct SSkipListKeyInfo { uint8_t dupKey : 2; // if allow duplicated key in the skip list - uint8_t type : 6; // key type + uint8_t type : 4; // key type + uint8_t freeNode:2; // free node when destroy the skiplist uint8_t len; // maximum key length, used in case of string key } SSkipListKeyInfo; typedef struct SSkipList { - __compar_fn_t comparFn; - __sl_key_fn_t keyFn; - uint32_t size; - uint8_t maxLevel; - uint8_t level; - SSkipListKeyInfo keyInfo; - + __compar_fn_t comparFn; + __sl_key_fn_t keyFn; + uint32_t size; + uint8_t maxLevel; + uint8_t level; + SSkipListKeyInfo keyInfo; pthread_rwlock_t *lock; - SSkipListNode * pHead; - + SSkipListNode * pHead; // point to the first element + SSkipListNode * pTail; // point to the last element + void * lastKey; // last key in the skiplist #if SKIP_LIST_RECORD_PERFORMANCE tSkipListState state; // skiplist state #endif - } SSkipList; /* @@ -147,7 +147,7 @@ typedef struct SSkipListIterator { * @return */ SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe, - __sl_key_fn_t fn); + uint8_t freeNode, __sl_key_fn_t fn); /** * @@ -182,21 +182,28 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode); * @param keyType * @return */ -SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType); +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType); /** * get the size of skip list * @param pSkipList * @return */ -size_t tSkipListGetSize(const SSkipList* pSkipList); +size_t tSkipListGetSize(const SSkipList *pSkipList); + +/** + * display skip list of the given level, for debug purpose only + * @param pSkipList + * @param nlevel + */ +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); /** * create skiplist iterator * @param pSkipList * @return */ -SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList); +SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); /** * forward the skip list iterator @@ -217,7 +224,7 @@ SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); * @param iter * @return */ -void* tSkipListDestroyIter(SSkipListIterator* iter); +void *tSkipListDestroyIter(SSkipListIterator *iter); /* * remove only one node of the pKey value. @@ -234,7 +241,6 @@ bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey); */ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); - #ifdef __cplusplus } #endif diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index df97dde5ac7b1440b2cdf2abab9f0243f84ddee5..b80aad1cebab3782ad7e2c4751275d854ef5f92c 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -176,6 +176,14 @@ uint32_t ip2uint(const char *const ip_addr); void taosSetAllocMode(int mode, const char* path, bool autoDump); void taosDumpMemoryLeak(); +#define TD_EQ 0x1 +#define TD_GT 0x2 +#define TD_LT 0x4 +#define TD_GE (TD_EQ | TD_GT) +#define TD_LE (TD_EQ | TD_LT) +void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, + int (*compar)(const void *, const void *), int flags); + #ifdef TAOS_MEM_CHECK void * taos_malloc(size_t size, const char *file, uint32_t line); diff --git a/src/sdb/src/hashint.c b/src/util/src/hashint.c similarity index 100% rename from src/sdb/src/hashint.c rename to src/util/src/hashint.c diff --git a/src/sdb/src/hashstr.c b/src/util/src/hashstr.c similarity index 100% rename from src/sdb/src/hashstr.c rename to src/util/src/hashstr.c diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index fb2dac827e13d4526b6b7a6c358a34dd575eb941..eb1d2133174b59a0ba93830a51986cbc78dc4c4b 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -128,6 +128,38 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { return dst; } +void taosArrayRemove(SArray* pArray, size_t index) { + assert(index < pArray->size); + + if (index == pArray->size - 1) { + taosArrayPop(pArray); + return; + } + + size_t remain = pArray->size - index - 1; + memmove(pArray->pData + index * pArray->elemSize, pArray->pData + (index + 1) * pArray->elemSize, remain * pArray->elemSize); + pArray->size -= 1; +} + +void taosArrayCopy(SArray* pDst, SArray* pSrc) { + assert(pSrc != NULL && pDst != NULL); + + if (pDst->capacity < pSrc->size) { + void* pData = realloc(pDst->pData, pSrc->size * pSrc->elemSize); + if (pData == NULL) { // todo handle oom + + } else { + pDst->pData = pData; + pDst->capacity = pSrc->size; + } + } + + memcpy(pDst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); + pDst->elemSize = pSrc->elemSize; + pDst->capacity = pSrc->size; + pDst->size = pSrc->size; +} + void taosArrayDestroy(SArray* pArray) { if (pArray == NULL) { return; diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index bbea30207cfbb275caf37673afd925bea63559d7..02de88881d9dcfdc2b0ac45114bd59036faa4d26 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -112,13 +112,7 @@ int tsReplications = TSDB_REPLICA_MIN_NUM; int tsNumOfMPeers = 3; int tsMaxShellConns = 2000; -int tsMaxAccounts = 100; -int tsMaxUsers = 1000; -int tsMaxDbs = 1000; -int tsMaxTables = 650000; -int tsMaxDnodes = 1000; -int tsMaxVGroups = 1000; -char tsMgmtZone[16] = "rzone"; +int tsMaxTables = 100000; char tsLocalIp[TSDB_IPv4ADDR_LEN] = {0}; char tsDefaultDB[TSDB_DB_NAME_LEN] = {0}; @@ -612,28 +606,10 @@ static void doInitGlobalConfig() { 1, 8640000, 0, TSDB_CFG_UTYPE_SECOND); // mgmt configs - tsInitConfigOption(cfg++, "mgmtZone", tsMgmtZone, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 0, 0, 16, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxAccounts", &tsMaxAccounts, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxUsers", &tsMaxUsers, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDbs", &tsMaxDbs, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 10000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxTables", &tsMaxTables, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 100000000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDnodes", &tsMaxDnodes, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxVGroups", &tsMaxVGroups, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 1000000, 0, TSDB_CFG_UTYPE_NONE); - + tsInitConfigOption(cfg++, "minSlidingTime", &tsMinSlidingTime, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 1000000, 0, TSDB_CFG_UTYPE_MS); diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index c50c38aa3cabd74266d4164f0ebdf4581ce670f6..5ed6b8cf232eb1fb774af75a9738f6a403cda93c 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -15,88 +15,81 @@ #include "os.h" #include "tlog.h" +#include typedef struct { int maxId; int numOfFree; int freeSlot; - int * freeList; + bool * freeList; pthread_mutex_t mutex; } id_pool_t; void *taosInitIdPool(int maxId) { - id_pool_t *pIdPool; - int * idList, i; - - if (maxId < 3) maxId = 3; - - pIdPool = (id_pool_t *)malloc(sizeof(id_pool_t)); + id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; - idList = (int *)malloc(sizeof(int) * (size_t)maxId); - if (idList == NULL) { + pIdPool->freeList = calloc(maxId, sizeof(bool)); + if (pIdPool->freeList == NULL) { free(pIdPool); return NULL; } - memset(pIdPool, 0, sizeof(id_pool_t)); pIdPool->maxId = maxId; - pIdPool->numOfFree = maxId - 1; + pIdPool->numOfFree = maxId; pIdPool->freeSlot = 0; - pIdPool->freeList = idList; pthread_mutex_init(&pIdPool->mutex, NULL); - for (i = 1; i < maxId; ++i) idList[i - 1] = i; - pTrace("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId); - return (void *)pIdPool; + return pIdPool; } int taosAllocateId(void *handle) { - id_pool_t *pIdPool; - int id = -1; - if (handle == NULL) return id; - - pIdPool = (id_pool_t *)handle; - - if (pIdPool->maxId < 3) pError("pool:%p is messed up, maxId:%d", pIdPool, pIdPool->maxId); + id_pool_t *pIdPool = handle; + if (handle == NULL) { + return -1; + } - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + int slot = -1; + pthread_mutex_lock(&pIdPool->mutex); if (pIdPool->numOfFree > 0) { - id = pIdPool->freeList[pIdPool->freeSlot]; - pIdPool->freeSlot = (pIdPool->freeSlot + 1) % pIdPool->maxId; - pIdPool->numOfFree--; + for (int i = 0; i < pIdPool->maxId; ++i) { + slot = (i + pIdPool->freeSlot) % pIdPool->maxId; + if (!pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = true; + pIdPool->freeSlot = slot + 1; + pIdPool->numOfFree--; + break; + } + } } - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); - - return id; + pthread_mutex_unlock(&pIdPool->mutex); + return slot + 1; } void taosFreeId(void *handle, int id) { - id_pool_t *pIdPool; - int slot; + id_pool_t *pIdPool = handle; + if (handle == NULL) return; - pIdPool = (id_pool_t *)handle; - if (pIdPool->freeList == NULL || pIdPool->maxId == 0) return; - if (id <= 0 || id >= pIdPool->maxId) return; - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + pthread_mutex_lock(&pIdPool->mutex); - slot = (pIdPool->freeSlot + pIdPool->numOfFree) % pIdPool->maxId; - pIdPool->freeList[slot] = id; - pIdPool->numOfFree++; + int slot = (id - 1) % pIdPool->maxId; + if (pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = false; + pIdPool->numOfFree++; + } - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + pthread_mutex_unlock(&pIdPool->mutex); } void taosIdPoolCleanUp(void *handle) { - id_pool_t *pIdPool; + id_pool_t *pIdPool = handle; - if (handle == NULL) return; - pIdPool = (id_pool_t *)handle; + if (pIdPool == NULL) return; pTrace("pool:%p is cleaned", pIdPool); @@ -110,42 +103,21 @@ void taosIdPoolCleanUp(void *handle) { } int taosIdPoolNumOfUsed(void *handle) { - id_pool_t *pIdPool = (id_pool_t *)handle; - - return pIdPool->maxId - pIdPool->numOfFree - 1; -} - -void taosIdPoolReinit(void *handle) { - id_pool_t *pIdPool; - - pIdPool = (id_pool_t *)handle; - pIdPool->numOfFree = 0; - pIdPool->freeSlot = 0; - - for (int i = 0; i < pIdPool->maxId; ++i) pIdPool->freeList[i] = 0; + id_pool_t *pIdPool = handle; + return pIdPool->maxId - pIdPool->numOfFree; } -void taosIdPoolMarkStatus(void *handle, int id, int status) { - id_pool_t *pIdPool = (id_pool_t *)handle; - - pIdPool->freeList[id] = status; -} +void taosIdPoolMarkStatus(void *handle, int id) { + id_pool_t *pIdPool = handle; + pthread_mutex_lock(&pIdPool->mutex); -void taosIdPoolSetFreeList(void *handle) { - id_pool_t *pIdPool; - int pos = 0; - - pIdPool = (id_pool_t *)handle; - pIdPool->numOfFree = 0; - pIdPool->freeSlot = 0; - - for (int i = 1; i < pIdPool->maxId; ++i) { - if (pIdPool->freeList[i] == 0) { - pIdPool->freeList[pos] = i; - pIdPool->numOfFree++; - pos++; - } + int slot = (id - 1) % pIdPool->maxId; + if (!pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = true; + pIdPool->numOfFree--; } + + pthread_mutex_unlock(&pIdPool->mutex); } int taosUpdateIdPool(id_pool_t *handle, int maxId) { @@ -154,18 +126,14 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { return -1; } - int *idList, i; - idList = (int *)malloc(sizeof(int) * (size_t)maxId); + int *idList = calloc(maxId, sizeof(bool)); if (idList == NULL) { return -1; } - for (i = 1; i < maxId; ++i) { - idList[i - 1] = i; - } - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + pthread_mutex_lock(&pIdPool->mutex); - memcpy(idList, pIdPool->freeList, sizeof(int) * (size_t)pIdPool->maxId); + memcpy(idList, pIdPool->freeList, sizeof(bool) * pIdPool->maxId); pIdPool->numOfFree += (maxId - pIdPool->maxId); pIdPool->maxId = maxId; @@ -173,7 +141,7 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { pIdPool->freeList = idList; free(oldIdList); - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + pthread_mutex_unlock(&pIdPool->mutex); return 0; } diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index badcb7802f510b2978abace6b21a1098e1cdc44d..bdb12a59f9776eaf2924c6c08c35c1b204e2065a 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -100,6 +100,8 @@ SListNode *tdListPopHead(SList *list) { list->head = node->next; } list->numOfEles--; + node->next = NULL; + node->prev = NULL; return node; } @@ -113,6 +115,7 @@ SListNode *tdListPopTail(SList *list) { list->tail = node->prev; } list->numOfEles--; + node->next = node->prev = NULL; return node; } @@ -131,6 +134,7 @@ SListNode *tdListPopNode(SList *list, SListNode *node) { node->next->prev = node->prev; } list->numOfEles--; + node->next = node->prev = NULL; return node; } @@ -138,11 +142,10 @@ SListNode *tdListPopNode(SList *list, SListNode *node) { // Move all node elements from src to dst, the dst is assumed as an empty list void tdListMove(SList *src, SList *dst) { // assert(dst->eleSize == src->eleSize); - dst->numOfEles = src->numOfEles; - dst->head = src->head; - dst->tail = src->tail; - src->numOfEles = 0; - src->head = src->tail = NULL; + SListNode *node = NULL; + while ((node = tdListPopHead(src)) != NULL) { + tdListAppendNode(dst, node); + } } void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(target, node->data, list->eleSize); } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 85b1d3d206f07647c2fd68c9e61e7f8d7367f290..11e1e740e04669f4f486c4e21e61ac174f57cdbd 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -15,7 +15,6 @@ #include "os.h" #include "tlog.h" -// #include "tsdb.h" #include "tskiplist.h" #include "tutil.h" @@ -52,28 +51,27 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { level = 1; pSkipList->level = 1; } else { - if (level > pSkipList->level && pSkipList->level < pSkipList->maxLevel) { - level = (++pSkipList->level); + if (level > pSkipList->level) { + if (pSkipList->level < pSkipList->maxLevel) { + level = (++pSkipList->level); + } else { + level = pSkipList->level; + } } } + + assert(level <= pSkipList->maxLevel); return level; } -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, int32_t level, SSkipListNode *pNode); - -void tSkipListDoRecordPut(SSkipList *pSkipList) { -#if SKIP_LIST_RECORD_PERFORMANCE - const int32_t MAX_RECORD_NUM = 1000; +#define DO_MEMSET_PTR_AREA(n) do {\ +int32_t _l = (n)->level;\ +memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\ +(n)->level = _l;\ +} while(0) - if (pSkipList->state.nInsertObjs == MAX_RECORD_NUM) { - pSkipList->state.nInsertObjs = 1; - pSkipList->state.nTotalStepsForInsert = 0; - pSkipList->state.nTotalElapsedTimeForInsert = 0; - } else { - pSkipList->state.nInsertObjs++; - } -#endif -} +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); +static SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode); int32_t compareInt32Val(const void *pLeft, const void *pRight) { int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); @@ -143,28 +141,12 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { } int32_t compareStrVal(const void *pLeft, const void *pRight) { - // SSkipListKey *pL = (SSkipListKey *)pLeft; - // SSkipListKey *pR = (SSkipListKey *)pRight; - // - // if (pL->nLen == 0 && pR->nLen == 0) { - // return 0; - // } - // - // // handle only one-side bound compare situation, there is only lower bound or only upper bound - // if (pL->nLen == -1) { - // return 1; // no lower bound, lower bound is minimum, always return -1; - // } else if (pR->nLen == -1) { - // return -1; // no upper bound, upper bound is maximum situation, always return 1; - // } - // - // int32_t ret = strcmp(((SSkipListKey *)pLeft)->pz, ((SSkipListKey *)pRight)->pz); - // - // if (ret == 0) { - // return 0; - // } else { - // return ret > 0 ? 1 : -1; - // } - return 0; + int32_t ret = strcmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } } int32_t compareWStrVal(const void *pLeft, const void *pRight) { @@ -281,8 +263,30 @@ static __compar_fn_t getKeyComparator(int32_t keyType) { return comparFn; } +static bool initForwardBackwardPtr(SSkipList* pSkipList) { + uint32_t maxLevel = pSkipList->maxLevel; + + // head info + pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel) * 2); + if (pSkipList->pHead == NULL) { + return false; + } + + pSkipList->pHead->level = pSkipList->maxLevel; + + // tail info + pSkipList->pTail = (SSkipListNode*) ((char*) pSkipList->pHead + SL_NODE_HEADER_SIZE(maxLevel)); + pSkipList->pTail->level = pSkipList->maxLevel; + + for(int32_t i = 0; i < maxLevel; ++i) { + SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; + SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; + } + + return true; +} SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t lock, - __sl_key_fn_t fn) { + uint8_t freeNode, __sl_key_fn_t fn) { SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); if (pSkipList == NULL) { return NULL; @@ -292,22 +296,24 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, ui maxLevel = MAX_SKIP_LIST_LEVEL; } - pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey}; - pSkipList->keyFn = fn; - + pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey, .freeNode = freeNode}; + pSkipList->keyFn = fn; pSkipList->comparFn = getKeyComparator(keyType); pSkipList->maxLevel = maxLevel; - pSkipList->level = 1; - - pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel)); - pSkipList->pHead->level = pSkipList->maxLevel; - + pSkipList->level = 1; + + if (!initForwardBackwardPtr(pSkipList)) { + tfree(pSkipList); + return NULL; + } + if (lock) { pSkipList->lock = calloc(1, sizeof(pthread_rwlock_t)); if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { tfree(pSkipList->pHead); tfree(pSkipList); + return NULL; } } @@ -349,16 +355,17 @@ void *tSkipListDestroy(SSkipList *pSkipList) { pthread_rwlock_wrlock(pSkipList->lock); } - SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); // pSkipList->pHead.pForward[0]; + SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); - while (pNode) { + while (pNode != pSkipList->pTail) { SSkipListNode *pTemp = pNode; pNode = SL_GET_FORWARD_POINTER(pNode, 0); - tfree(pTemp); + + if (pSkipList->keyInfo.freeNode) { + tfree(pTemp); + } } - tfree(pSkipList->pHead); - if (pSkipList->lock) { pthread_rwlock_unlock(pSkipList->lock); pthread_rwlock_destroy(pSkipList->lock); @@ -381,96 +388,110 @@ void tSkipListRandNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSi } SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { - if (pSkipList == NULL) { + if (pSkipList == NULL || pNode == NULL) { return NULL; } if (pSkipList->lock) { pthread_rwlock_wrlock(pSkipList->lock); } - - // record one node is put into skiplist - tSkipListDoRecordPut(pSkipList); - + + // the new key is greater than the last key of skiplist append it at last position + char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); + if (pSkipList->size == 0 || pSkipList->comparFn(pSkipList->lastKey, newDatakey) < 0) { + return tSkipListDoAppend(pSkipList, pNode); + } + + // find the appropriated position to insert data SSkipListNode *px = pSkipList->pHead; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; + bool identical = false; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); - while (p != NULL) { + while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); - char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); // if the forward element is less than the specified key, forward one step - if (pSkipList->comparFn(key, newDatakey) < 0) { + int32_t ret = pSkipList->comparFn(key, newDatakey); + if (ret < 0) { px = p; - p = SL_GET_FORWARD_POINTER(px, i); } else { + if (identical == false) { + identical = (ret == 0); + } + break; } } - -#if SKIP_LIST_RECORD_PERFORMANCE - pSkipList->state.nTotalStepsForInsert++; -#endif + forward[i] = px; } // if the skip list does not allowed identical key inserted, the new data will be discarded. - if (pSkipList->keyInfo.dupKey == 0 && forward[0] != pSkipList->pHead) { - char *key = SL_GET_NODE_KEY(pSkipList, forward[0]); - char *pNewDataKey = SL_GET_NODE_KEY(pSkipList, pNode); - - if (pSkipList->comparFn(key, pNewDataKey) == 0) { - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return forward[0]; + if (pSkipList->keyInfo.dupKey == 0 && identical) { + if (pSkipList->lock) { + pthread_rwlock_unlock(pSkipList->lock); } - } - -#if SKIP_LIST_RECORD_PERFORMANCE - recordNodeEachLevel(pSkipList, level); -#endif - - int32_t level = SL_GET_NODE_LEVEL(pNode); - tSkipListDoInsert(pSkipList, forward, level, pNode); - atomic_add_fetch_32(&pSkipList->size, 1); - -#if SKIP_LIST_RECORD_PERFORMANCE - pSkipList->state.nTotalMemSize += getOneNodeSize(pKey, level); -#endif - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); + return forward[0]; } - + + tSkipListDoInsert(pSkipList, forward, pNode); return pNode; } -void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, int32_t level, SSkipListNode *pNode) { - for (int32_t i = 0; i < level; ++i) { +void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { + DO_MEMSET_PTR_AREA(pNode); + + for (int32_t i = 0; i < pNode->level; ++i) { SSkipListNode *x = forward[i]; - if (x != NULL) { + +// if (x != pSkipList->pTail) { SL_GET_BACKWARD_POINTER(pNode, i) = x; - SSkipListNode *pForward = SL_GET_FORWARD_POINTER(x, i); - if (pForward) { - SL_GET_BACKWARD_POINTER(pForward, i) = pNode; - } + SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); +// if (next) { + SL_GET_BACKWARD_POINTER(next, i) = pNode; +// } - SL_GET_FORWARD_POINTER(pNode, i) = SL_GET_FORWARD_POINTER(x, i); + SL_GET_FORWARD_POINTER(pNode, i) = next; SL_GET_FORWARD_POINTER(x, i) = pNode; - } else { - SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; - SL_GET_BACKWARD_POINTER(pSkipList->pHead, i) = (pSkipList->pHead); - } +// } else { +// SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; +// } + } + + atomic_add_fetch_32(&pSkipList->size, 1); + if (pSkipList->lock) { + pthread_rwlock_unlock(pSkipList->lock); } } +SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode) { + // do clear pointer area + DO_MEMSET_PTR_AREA(pNode); + + for(int32_t i = 0; i < pNode->level; ++i) { + SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i); + SL_GET_FORWARD_POINTER(prev, i) = pNode; + SL_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; + + SL_GET_BACKWARD_POINTER(pNode, i) = prev; + SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; + } + + pSkipList->lastKey = SL_GET_NODE_KEY(pSkipList, pNode); + + atomic_add_fetch_32(&pSkipList->size, 1); + if (pSkipList->lock) { + pthread_rwlock_unlock(pSkipList->lock); + } + + return pNode; +} + SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) { int32_t sLevel = pSkipList->level - 1; int32_t ret = -1; @@ -691,89 +712,6 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // return NULL; //} // -// int32_t tSkipListIterateList(SSkipList *pSkipList, SSkipListNode ***pRes, bool (*fp)(SSkipListNode *, void *), -// void *param) { -// (*pRes) = (SSkipListNode **)calloc(1, POINTER_BYTES * pSkipList->nSize); -// if (NULL == *pRes) { -// pError("error skiplist %p, malloc failed", pSkipList); -// return -1; -// } -// -// pthread_rwlock_rdlock(&pSkipList->lock); -// SSkipListNode *pStartNode = pSkipList->pHead.pForward[0]; -// int32_t num = 0; -// -// for (int32_t i = 0; i < pSkipList->nSize; ++i) { -// if (pStartNode == NULL) { -// pError("error skiplist %p, required length:%d, actual length:%d", pSkipList, pSkipList->nSize, i - 1); -//#ifdef _DEBUG_VIEW -// tSkipListPrint(pSkipList, 1); -//#endif -// break; -// } -// -// if (fp == NULL || (fp != NULL && fp(pStartNode, param) == true)) { -// (*pRes)[num++] = pStartNode; -// } -// -// pStartNode = pStartNode->pForward[0]; -// } -// -// pthread_rwlock_unlock(&pSkipList->lock); -// -// if (num == 0) { -// free(*pRes); -// *pRes = NULL; -// } else if (num < pSkipList->nSize) { // free unused memory -// char *tmp = realloc((*pRes), num * POINTER_BYTES); -// assert(tmp != NULL); -// -// *pRes = (SSkipListNode **)tmp; -// } -// -// return num; -//} -// -// int32_t tSkipListIteratorReset(SSkipList *pSkipList, SSkipListIterator *iter) { -// if (pSkipList == NULL) { -// return -1; -// } -// -// iter->pSkipList = pSkipList; -// if (pSkipList->lock) { -// pthread_rwlock_rdlock(&pSkipList->lock); -// } -// iter->cur = NULL; // pSkipList->pHead.pForward[0]; -// iter->num = pSkipList->size; -// -// if (pSkipList->lock) { -// pthread_rwlock_unlock(&pSkipList->lock); -// } -// -// return 0; -//} -// -// bool tSkipListIteratorNext(SSkipListIterator *iter) { -// if (iter->num == 0 || iter->pSkipList == NULL) { -// return false; -// } -// -// SSkipList *pSkipList = iter->pSkipList; -// -// pthread_rwlock_rdlock(&pSkipList->lock); -// if (iter->cur == NULL) { -// iter->cur = pSkipList->pHead.pForward[0]; -// } else { -// iter->cur = iter->cur->pForward[0]; -// } -// -// pthread_rwlock_unlock(&pSkipList->lock); -// -// return iter->cur != NULL; -//} -// -// SSkipListNode *tSkipListIteratorGet(SSkipListIterator *iter) { return iter->cur; } -// // int32_t tSkipListRangeQuery(SSkipList *pSkipList, tSKipListQueryCond *pCond, SSkipListNode ***pRes) { // pSkipList->state.queryCount++; // SSkipListNode *pStart = tSkipListParQuery(pSkipList, &pCond->lowerBnd, pCond->lowerBndRelOptr); @@ -841,12 +779,20 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { } SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); - int32_t id = 1; - - while (p) { + + int32_t id = 1; + char* prev = NULL; + + while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); + if (prev != NULL) { + assert(pSkipList->comparFn(prev, key) < 0); + } + switch (pSkipList->keyInfo.type) { case TSDB_DATA_TYPE_INT: + fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key); + break; case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_BIGINT: @@ -862,7 +808,8 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { fprintf(stdout, "\n"); } + prev = SL_GET_NODE_KEY(pSkipList, p); + p = SL_GET_FORWARD_POINTER(p, nlevel - 1); - // p = p->pForward[nlevel - 1]; } } diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 9c384b25bace047cdd80fe2903c330dbe3ed85d7..cbd08954cc21ffd1f8a0e3cfc9076e0330860c42 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -617,3 +617,72 @@ char *taosCharsetReplace(char *charsetstr) { return strdup(charsetstr); } + +#define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx)) +void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, int (*compar)(const void *, const void *), int flags) { + // TODO: need to check the correctness of this function + int l = 0; + int r = nmemb; + int idx = 0; + int comparison; + + if (flags == TD_EQ) { + return bsearch(key, base, nmemb, size, compar); + } else if (flags == TD_GE) { + if ((*compar)(key, elePtrAt(base, size, 0)) <= 0) return elePtrAt(base, size, 0); + if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) > 0) return NULL; + + while (l < r) { + idx = (l + r) / 2; + comparison = (*compar)(key, elePtrAt(base, size, idx)); + if (comparison < 0) { + r = idx; + } else if (comparison > 0) { + l = idx + 1; + } else { + return elePtrAt(base, size, idx); + } + } + + if ((*compar)(key, elePtrAt(base, size, idx) < 0)) { + return elePtrAt(base, size, idx); + } else { + if (idx + 1 > nmemb - 1) { + return NULL; + } else { + return elePtrAt(base, size, idx + 1); + } + } + } else if (flags == TD_LE) { + if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) >= 0) return elePtrAt(base, size, nmemb - 1); + if ((*compar)(key, elePtrAt(base, size, 0)) < 0) return NULL; + + while (l < r) { + idx = (l + r) / 2; + comparison = (*compar)(key, elePtrAt(base, size, idx)); + if (comparison < 0) { + r = idx; + } else if (comparison > 0) { + l = idx + 1; + } else { + return elePtrAt(base, size, idx); + } + } + + if ((*compar)(key, elePtrAt(base, size, idx)) > 0) { + return elePtrAt(base, size, idx); + } else { + if (idx == 0) { + return NULL; + } else { + return elePtrAt(base, size, idx - 1); + } + } + + } else { + assert(0); + return NULL; + } + + return NULL; +} diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..042c925165985a285670f5df845708f92146b353 --- /dev/null +++ b/src/util/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) + +IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) + MESSAGE(STATUS "gTest library found, build unit test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(utilTest tutil gtest pthread) +ENDIF() \ No newline at end of file diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1902fef4b64d0b796a85d9f58e1f422fe0df59e8 --- /dev/null +++ b/src/util/tests/cacheTest.cpp @@ -0,0 +1,142 @@ +#include +#include +#include + +#include "taos.h" +//#include "tsdb.h" + +//#include "testCommon.h" +#include "tstoken.h" +#include "tutil.h" +#include "tcache.h" +#include "ttimer.h" +#include "ttime.h" + +namespace { +int32_t tsMaxMgmtConnections = 10000; +int32_t tsMaxMeterConnections = 200; +} +// test cache +TEST(testCase, client_cache_test) { + const int32_t REFRESH_TIME_IN_SEC = 2; + void* tscTmr = taosTmrInit (tsMaxMgmtConnections*2, 200, 6000, "TSC"); + SCacheObj* tscCacheHandle = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC); + + char* key1 = "test1"; + char* data1 = "test11"; + + char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1), 1); + sleep(REFRESH_TIME_IN_SEC+1); + + printf("obj is still valid: %s\n", cachedObj); + + char* data2 = "test22"; + taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + + /* the object is cleared by cache clean operation */ + cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2), 20); + printf("after updated: %s\n", cachedObj); + + printf("start to remove data from cache\n"); + taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + printf("end of removing data from cache\n"); + + getchar(); + + char* key3 = "test2"; + char* data3 = "kkkkkkk"; + + char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3), 1); + printf("%s\n", cachedObj2); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + + sleep(3); + char* d = (char*) taosCacheAcquireByName(tscCacheHandle, key3); +// assert(d == NULL); + + char* key5 = "test5"; + char* data5 = "data5kkkkk"; + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5), 20); + + char* data6= "new Data after updated"; + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6), 20); + printf("%s\n", cachedObj2); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); + + char* data7 = "add call update procedure"; + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7), 20); + printf("%s\n=======================================\n\n", cachedObj2); + + char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); + taosCacheRelease(tscCacheHandle, (void**) &cc, false); + + char* data8 = "ttft"; + char* key6 = "key6"; + + char* ft = (char*) taosCachePut(tscCacheHandle, key6, data8, strlen(data8), 20); + taosCacheRelease(tscCacheHandle, (void**) &ft, false); + + /** + * 140ns + */ + uint64_t startTime = taosGetTimestampUs(); + printf("Cache Performance Test\nstart time:%lld\n", startTime); + for(int32_t i=0; i<1000; ++i) { + char* dd = (char*) taosCacheAcquireByName(tscCacheHandle, key6); + if (dd != NULL) { +// printf("get the data\n"); + } else { + printf("data has been released\n"); + } + + taosCacheRelease(tscCacheHandle, (void**) &dd, false); + } + + uint64_t endTime = taosGetTimestampUs(); + int64_t el = endTime - startTime; + + printf("End of Test, %lld\nTotal Elapsed Time:%lld us.avg:%f us\n", endTime, el, el/1000.0); + + taosCacheCleanup(tscCacheHandle); +} + +TEST(testCase, cache_resize_test) { + const int32_t REFRESH_TIME_IN_SEC = 2; + void* tscTmr = taosTmrInit (1000*2, 200, 6000, "TSC"); + + auto* pCache = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC); + + char key[256] = {0}; + char data[1024] = "abcdefghijk"; + int32_t len = strlen(data); + + uint64_t startTime = taosGetTimestampUs(); + int32_t num = 10000; + + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "abc_%7d", i); + taosCachePut(pCache, key, data, len, 3600); + } + uint64_t endTime = taosGetTimestampUs(); + + printf("add %d object cost:%lld us, avg:%f us\n", num, endTime - startTime, (endTime-startTime)/(double)num); + + startTime = taosGetTimestampUs(); + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "abc_%7d", i); + void* k = taosCacheAcquireByName(pCache, key); + assert(k != 0); + } + endTime = taosGetTimestampUs(); + printf("retrieve %d object cost:%lld us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num); + + taosCacheCleanup(pCache); + taosMsleep(20000); + getchar(); +} \ No newline at end of file diff --git a/src/util/tests/hashTest.cpp b/src/util/tests/hashTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b3baedb69692ab4322d6a033dc1c529f3034f905 --- /dev/null +++ b/src/util/tests/hashTest.cpp @@ -0,0 +1,156 @@ +#include +#include +#include +#include + +#include "hash.h" +#include "taos.h" +#include "ttime.h" + +namespace { +// the simple test code for basic operations +void simpleTest() { + auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + // put 400 elements in the hash table + for(int32_t i = -200; i < 200; ++i) { + taosHashPut(hashTable, (const char*) &i, sizeof(int32_t), (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 400); + + for(int32_t i = 0; i < 200; ++i) { + char* p = (char*) taosHashGet(hashTable, (const char*) &i, sizeof(int32_t)); + ASSERT_TRUE(p != nullptr); + ASSERT_EQ(*reinterpret_cast(p), i); + } + + for(int32_t i = 1000; i < 2000; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 400); + + for(int32_t i = 0; i < 100; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 300); + + for(int32_t i = 100; i < 150; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 250); + taosHashCleanup(hashTable); +} + +void stringKeyTest() { + auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + char key[128] = {0}; + + // put 200 elements in the hash table + for(int32_t i = 0; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 1000); + + for(int32_t i = 0; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + char* p = (char*) taosHashGet(hashTable, key, len); + ASSERT_TRUE(p != nullptr); + + ASSERT_EQ(*reinterpret_cast(p), i); + } + + for(int32_t i = 500; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + taosHashRemove(hashTable, key, len); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 500); + + for(int32_t i = 0; i < 499; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + taosHashRemove(hashTable, key, len); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 1); + + taosHashCleanup(hashTable); +} + +void functionTest() { + +} + +/** + * evaluate the performance issue, by add 10million elements in to hash table in + * a single threads situation + */ +void noLockPerformanceTest() { + auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + char key[128] = {0}; + int32_t num = 5000000; + + int64_t st = taosGetTimestampUs(); + + // put 10M elements in the hash table + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), num); + + int64_t et = taosGetTimestampUs(); + printf("Elpased time:%" PRId64 " us to add %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num); + + st = taosGetTimestampUs(); + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + char* p = (char*) taosHashGet(hashTable, key, len); + ASSERT_TRUE(p != nullptr); + + ASSERT_EQ(*reinterpret_cast(p), i); + } + + et = taosGetTimestampUs(); + printf("Elpased time:%" PRId64 " us to fetch all %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num); + + printf("The maximum length of overflow linklist in hash table is:%d\n", taosHashGetMaxOverflowLinkLength(hashTable)); + taosHashCleanup(hashTable); +} + +void multithreadsTest() { + //todo +} + +// check the function robustness +void invalidOperationTest() { + +} + +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, hashTest) { +// simpleTest(); +// stringKeyTest(); +// noLockPerformanceTest(); +// multithreadsTest(); +} \ No newline at end of file diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c009231874de94530ffa18d15f745531c4daa94c --- /dev/null +++ b/src/util/tests/skiplistTest.cpp @@ -0,0 +1,355 @@ +#include +#include +#include +#include + +#include "taosmsg.h" +#include "tskiplist.h" +#include "ttime.h" +#include "tutil.h" + +namespace { + +char* getkey(const void* data) { return (char*)(data); } + +void doubleSkipListTest() { + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, true, getkey); + + double doubleVal[1000] = {0}; + int32_t size = 20000; + + printf("generated %d keys is: \n", size); + + for (int32_t i = 0; i < size; ++i) { + if (i < 1000) { + doubleVal[i] = i * 0.997; + } + + int32_t level = 0; + int32_t size = 0; + + tSkipListRandNodeInfo(pSkipList, &level, &size); + auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2); + d->level = level; + + double* key = (double*)SL_GET_NODE_KEY(pSkipList, d); + key[0] = i * 0.997; + key[1] = i * 0.997; + + tSkipListPut(pSkipList, d); + } + + printf("the first level of skip list is:\n"); + tSkipListPrint(pSkipList, 1); + +#if 0 + SSkipListNode **pNodes = NULL; + SSkipListKey sk; + for (int32_t i = 0; i < 100; ++i) { + sk.nType = TSDB_DATA_TYPE_DOUBLE; + int32_t idx = abs((i * rand()) % 1000); + + sk.dKey = doubleVal[idx]; + + int32_t size = tSkipListGets(pSkipList, &sk, &pNodes); + + printf("the query result size is: %d\n", size); + for (int32_t j = 0; j < size; ++j) { + printf("the result is: %lf\n", pNodes[j]->key.dKey); + } + + if (size > 0) { + tfree(pNodes); + } + } + +#endif + + printf("double test end...\n"); + tSkipListDestroy(pSkipList); +} + +void randKeyTest() { + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_INT, sizeof(int32_t), 0, false, true, getkey); + + int32_t size = 200000; + srand(time(NULL)); + + printf("generated %d keys is: \n", size); + + for (int32_t i = 0; i < size; ++i) { + int32_t level = 0; + int32_t s = 0; + + tSkipListRandNodeInfo(pSkipList, &level, &s); + auto d = (SSkipListNode*)calloc(1, s + sizeof(int32_t) * 2); + d->level = level; + + int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); + key[0] = rand() % 1000000000; + + key[1] = key[0]; + + tSkipListPut(pSkipList, d); + } + + printf("the first level of skip list is:\n"); + tSkipListPrint(pSkipList, 1); + + printf("the sec level of skip list is:\n"); + tSkipListPrint(pSkipList, 2); + + printf("the 5 level of skip list is:\n"); + tSkipListPrint(pSkipList, 5); + + tSkipListDestroy(pSkipList); +} +void stringKeySkiplistTest() { + const int32_t max_key_size = 12; + + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_BINARY, max_key_size, 0, false, true, getkey); + + int32_t level = 0; + int32_t headsize = 0; + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + auto pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + pNode->level = level; + + char* d = SL_GET_NODE_DATA(pNode); + strncpy(d, "nyse", 5); + + *(double*)(d + max_key_size) = 12; + + tSkipListPut(pSkipList, pNode); + + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + pNode->level = level; + + d = SL_GET_NODE_DATA(pNode); + strncpy(d, "beijing", 8); + + *(double*)(d + max_key_size) = 911; + + tSkipListPut(pSkipList, pNode); + + printf("level one------------------\n"); + tSkipListPrint(pSkipList, 1); + +#if 0 + SSkipListNode **pRes = NULL; + int32_t ret = tSkipListGets(pSkipList, &key1, &pRes); + + assert(ret == 1); + assert(strcmp(pRes[0]->key.pz, "beijing") == 0); + assert(pRes[0]->key.nType == TSDB_DATA_TYPE_BINARY); + + tSkipListDestroyKey(&key1); + tSkipListDestroyKey(&key); + + tSkipListDestroy(pSkipList); + + free(pRes); +#endif + + tSkipListDestroy(pSkipList); + + int64_t s = taosGetTimestampUs(); + pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_BINARY, 20, 0, false, true, getkey); + char k[256] = {0}; + + int32_t total = 10000; + for (int32_t i = 0; i < total; ++i) { + int32_t n = sprintf(k, "abc_%d_%d", i, i); + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + auto pNode = (SSkipListNode*)calloc(1, headsize + 20 + sizeof(double)); + pNode->level = level; + + char* d = SL_GET_NODE_DATA(pNode); + strncpy(d, k, strlen(k)); + + tSkipListPut(pSkipList, pNode); + } + + int64_t e = taosGetTimestampUs(); + printf("elapsed time:%lld us to insert %d data, avg:%f us\n", (e - s), total, (double)(e - s) / total); + + printf("level two------------------\n"); + tSkipListPrint(pSkipList, 1); + +#if 0 + SSkipListNode **pres = NULL; + + s = taosGetTimestampMs(); + for (int32_t j = 0; j < total; ++j) { + int32_t n = sprintf(k, "abc_%d_%d", j, j); + key = tSkipListCreateKey(TSDB_DATA_TYPE_BINARY, k, n); + + int32_t num = tSkipListGets(pSkipList, &key, &pres); + assert(num > 0); + + // tSkipListRemove(pSkipList, &key); + tSkipListRemoveNode(pSkipList, pres[0]); + + if (num > 0) { + tfree(pres); + } + } + + e = taosGetTimestampMs(); + printf("elapsed time:%lldms\n", e - s); +#endif + tSkipListDestroy(pSkipList); +} + +void skiplistPerformanceTest() { + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, false, getkey); + + int32_t size = 1000000; + int64_t prev = taosGetTimestampMs(); + int64_t s = prev; + + int32_t level = 0; + int32_t headsize = 0; + + int32_t unit = MAX_SKIP_LIST_LEVEL * POINTER_BYTES * 2 + sizeof(double) * 2 + sizeof(int16_t); + + char* total = (char*)calloc(1, unit * size); + char* p = total; + + for (int32_t i = 0; i < size; ++i) { + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + SSkipListNode* d = (SSkipListNode*)p; + p += headsize + sizeof(double) * 2; + + d->level = level; + double* v = (double*)SL_GET_NODE_DATA(d); + v[0] = i * 0.997; + v[1] = i * 0.997; + + tSkipListPut(pSkipList, d); + + if (i % 100000 == 0) { + int64_t cur = taosGetTimestampMs(); + + int64_t elapsed = cur - prev; + printf("add %d, elapsed time: %lld ms, avg elapsed:%f ms, total:%d\n", 100000, elapsed, elapsed / 100000.0, i); + prev = cur; + } + } + + int64_t e = taosGetTimestampMs(); + printf("total:%lld ms, avg:%f\n", e - s, (e - s) / (double)size); + printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level); + + assert(tSkipListGetSize(pSkipList) == size); + + // printf("the level of skiplist is:\n"); + // + // printf("level two------------------\n"); + // tSkipListPrint(pSkipList, 2); + // + // printf("level three------------------\n"); + // tSkipListPrint(pSkipList, 3); + // + // printf("level four------------------\n"); + // tSkipListPrint(pSkipList, 4); + // + // printf("level nine------------------\n"); + // tSkipListPrint(pSkipList, 10); + + int64_t st = taosGetTimestampMs(); +#if 0 + for (int32_t i = 0; i < 100000; i += 1) { + key.dKey = i * 0.997; + tSkipListRemove(pSkipList, &key); + } +#endif + + int64_t et = taosGetTimestampMs(); + printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st); + assert(tSkipListGetSize(pSkipList) == size); + + tSkipListDestroy(pSkipList); + tfree(total); +} + +// todo not support duplicated key yet +void duplicatedKeyTest() { +#if 0 + SSkipListKey key; + key.nType = TSDB_DATA_TYPE_INT; + + SSkipListNode **pNodes = NULL; + + SSkipList *pSkipList = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_INT, sizeof(int)); + + for (int32_t i = 0; i < 10000; ++i) { + for (int32_t j = 0; j < 5; ++j) { + key.i64Key = i; + tSkipListPut(pSkipList, "", &key, 1); + } + } + + tSkipListPrint(pSkipList, 1); + + for (int32_t i = 0; i < 100; ++i) { + key.i64Key = rand() % 1000; + int32_t size = tSkipListGets(pSkipList, &key, &pNodes); + + assert(size == 5); + + tfree(pNodes); + } + + tSkipListDestroy(pSkipList); +#endif +} + +} // namespace + +TEST(testCase, skiplist_test) { + assert(sizeof(SSkipListKey) == 8); + srand(time(NULL)); + + stringKeySkiplistTest(); + doubleSkipListTest(); + skiplistPerformanceTest(); + duplicatedKeyTest(); + randKeyTest(); + + // tSKipListQueryCond q; + // q.upperBndRelOptr = true; + // q.lowerBndRelOptr = true; + // q.upperBnd.nType = TSDB_DATA_TYPE_DOUBLE; + // q.lowerBnd.nType = TSDB_DATA_TYPE_DOUBLE; + // q.lowerBnd.dKey = 120; + // q.upperBnd.dKey = 171.989; + /* + int32_t size = tSkipListQuery(pSkipList, &q, &pNodes); + for (int32_t i = 0; i < size; ++i) { + printf("-----%lf\n", pNodes[i]->key.dKey); + } + printf("the range query result size is: %d\n", size); + tfree(pNodes); + + SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20); + for (int32_t i = 0; i < 8; i += 2) { + pKeys[i].dKey = i * 0.997; + pKeys[i].nType = TSDB_DATA_TYPE_DOUBLE; + printf("%lf ", pKeys[i].dKey); + } + + int32_t r = tSkipListPointQuery(pSkipList, pKeys, 8, EXCLUDE_POINT_QUERY, &pNodes); + printf("\nthe exclude query result is: %d\n", r); + for (int32_t i = 0; i < r; ++i) { + // printf("%lf ", pNodes[i]->key.dKey); + } + tfree(pNodes); + + free(pKeys);*/ +} diff --git a/src/util/tests/stringTest.cpp b/src/util/tests/stringTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ef8df90e3eac8c4381cbc23cd5311b794b73d058 --- /dev/null +++ b/src/util/tests/stringTest.cpp @@ -0,0 +1,178 @@ +#include +#include +#include +#include + +#include "taos.h" +#include "tutil.h" + +TEST(testCase, string_dequote_test) { + char t1[] = "'abc'"; + int32_t len = strdequote(t1); + + EXPECT_EQ(3, len); + EXPECT_STRCASEEQ(t1, "abc"); + + char t2[] = "\"abc\""; + len = strdequote(t2); + + EXPECT_EQ(3, len); + EXPECT_STRCASEEQ(t1, "abc"); + + char t21[] = " abc "; + strtrim(t21); + + EXPECT_STREQ("abc", t21); + EXPECT_EQ(3, strlen(t21)); +} + +TEST(testCase, string_replace_test) { + char t3[] = "abc01abc02abc"; + char* ret = strreplace(t3, "abc", "7"); + + EXPECT_EQ(strlen(ret), 7); + EXPECT_STREQ("7017027", ret); + free(ret); + + char t4[] = "a01a02b03c04d05"; + ret = strreplace(t4, "0", "9999999999"); + + EXPECT_EQ(strlen(ret), 5 * 10 + 10); + EXPECT_STREQ("a99999999991a99999999992b99999999993c99999999994d99999999995", ret); + free(ret); + + char t5[] = "abc"; + ret = strreplace(t5, "abc", "12345678901234567890"); + + EXPECT_EQ(strlen(ret), 20); + EXPECT_STREQ("12345678901234567890", ret); + free(ret); + + char t6[] = "abc"; + ret = strreplace(t6, "def", "abc"); + + EXPECT_EQ(strlen(ret), 3); + EXPECT_STREQ("abc", ret); + free(ret); + + char t7[] = "abcde000000000000001234"; + ret = strreplace(t7, "ab", "0000000"); + + EXPECT_EQ(strlen(ret), 28); + EXPECT_STREQ("0000000cde000000000000001234", ret); + free(ret); + + char t8[] = "abc\ndef"; + char t[] = {10, 0}; + + char f1[] = "\\n"; + int32_t fx = strlen(f1); + ret = strreplace(t8, "\n", "\\n"); + + EXPECT_EQ(strlen(ret), 8); + EXPECT_STREQ("abc\\ndef", ret); + free(ret); + + char t9[] = "abc\\ndef"; + ret = strreplace(t9, "\\n", "\n"); + + EXPECT_EQ(strlen(ret), 7); + EXPECT_STREQ("abc\ndef", ret); + free(ret); + + char t10[] = "abcdef"; + ret = strreplace(t10, "", "0"); + + EXPECT_EQ(strlen(ret), 6); + EXPECT_STREQ("abcdef", ret); + free(ret); +} + +TEST(testCase, string_tolower_test) { + char t[1024] = {1}; + memset(t, 1, tListLen(t)); + + const char* a1 = "ABC"; + strtolower(t, a1); + EXPECT_STREQ(t, "abc"); + + memset(t, 1, tListLen(t)); + const char* a2 = "ABC\'ABC\'D"; + strtolower(t, a2); + EXPECT_STREQ(t, "abc\'ABC\'d"); + + memset(t, 1, tListLen(t)); + const char* a3 = ""; + strtolower(t, a3); + EXPECT_STREQ(t, ""); + + memset(t, 1, tListLen(t)); + const char* a4 = "\"AbcDEF\""; + strtolower(t, a4); + EXPECT_STREQ(t, a4); + + memset(t, 1, tListLen(t)); + const char* a5 = "1234\"AbcDEF\"456"; + strtolower(t, a5); + EXPECT_STREQ(t, a5); + + memset(t, 1, tListLen(t)); + const char* a6 = "1234"; + strtolower(t, a6); + EXPECT_STREQ(t, a6); +} + +TEST(testCase, string_strnchr_test) { + char t[1024] = {0}; + memset(t, 1, tListLen(t)); + + char a1[] = "AB.C"; + EXPECT_TRUE(strnchr(a1, '.', strlen(a1), true) != NULL); + + char a2[] = "abc."; + EXPECT_TRUE(strnchr(a2, '.', strlen(a2), true) != NULL); + + char a8[] = "abc."; + EXPECT_TRUE(strnchr(a8, '.', 1, true) == NULL); + + char a3[] = ".abc"; + EXPECT_TRUE(strnchr(a3, '.', strlen(a3), true) != NULL); + + char a4[] = "'.abc'"; + EXPECT_TRUE(strnchr(a4, '.', strlen(a4), true) == NULL); + + char a5[] = "'.abc.'abc"; + EXPECT_TRUE(strnchr(a5, '.', strlen(a5), true) == NULL); + + char a6[] = "0123456789."; + EXPECT_TRUE(strnchr(a6, '.', strlen(a6), true) != NULL); + + char a7[] = "0123456789."; + EXPECT_TRUE(strnchr(a7, '.', 3, true) == NULL); + + char a9[] = "0123456789."; + EXPECT_TRUE(strnchr(a9, '.', 0, true) == NULL); + + char a10[] = "0123456789'.'"; + EXPECT_TRUE(strnchr(a10, '.', strlen(a10), true) == NULL); +} + +// TEST(testCase, cache_resize_test) { +// char a11[] = "abc'.'"; +// EXPECT_TRUE(strnchr(a11, '.', strlen(a11), false) != NULL); + +// char a12[] = "abc'-'"; +// EXPECT_TRUE(strnchr(a12, '-', strlen(a12), false) != NULL); + +// char a15[] = "abc'-'"; +// EXPECT_TRUE(strnchr(a15, '-', strlen(a15), true) == NULL); + +// char a13[] = "'-'"; +// EXPECT_TRUE(strnchr(a13, '-', strlen(a13), false) != NULL); + +// char a14[] = "'-'"; +// EXPECT_TRUE(strnchr(a14, '-', strlen(a14), true) == NULL); + +// char a16[] = "'-'."; +// EXPECT_TRUE(strnchr(a16, '.', strlen(a16), true) != NULL); +// } \ No newline at end of file diff --git a/src/vnode/detail/src/vnodeQueryImpl.c b/src/vnode/detail/src/vnodeQueryImpl.c index 9eb3fb8b65f8f6299729b77edf4c14776f0d99f9..f3e5cc27b3105a6e5df1b43b934c1f9eed3afa74 100644 --- a/src/vnode/detail/src/vnodeQueryImpl.c +++ b/src/vnode/detail/src/vnodeQueryImpl.c @@ -5312,7 +5312,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (1) { // check if query is killed or not set the status of query to pass the status check - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return cnt; } @@ -6375,7 +6375,7 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->scanFlag = REPEAT_SCAN; /* check if query is killed or not */ - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } diff --git a/src/vnode/detail/src/vnodeQueryProcess.c b/src/vnode/detail/src/vnodeQueryProcess.c index cedb76b4accda46e934694cdbb6de416dcc8f75b..23520f35a168dbf9f80c8c5ff0714ab4557ade61 100644 --- a/src/vnode/detail/src/vnodeQueryProcess.c +++ b/src/vnode/detail/src/vnodeQueryProcess.c @@ -105,7 +105,7 @@ static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo int32_t start = pSupporter->pSidSet->starterPos[groupIdx]; int32_t end = pSupporter->pSidSet->starterPos[groupIdx + 1] - 1; - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } @@ -276,7 +276,7 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo int64_t st = taosGetTimestampUs(); while (1) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { break; } @@ -363,7 +363,7 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo int32_t j = QUERY_IS_ASC_QUERY(pQuery) ? 0 : numOfBlocks - 1; for (; j < numOfBlocks && j >= 0; j += step) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { break; } @@ -603,7 +603,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pSupporter->meterIdx = start; for (int32_t k = start; k <= end; ++k, pSupporter->meterIdx++) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -630,7 +630,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pSupporter->subgroupIdx); for (int32_t k = start; k <= end; ++k) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -681,7 +681,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { while (pSupporter->meterIdx < pSupporter->numOfMeters) { int32_t k = pSupporter->meterIdx; - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -958,7 +958,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { doMultiMeterSupplementaryScan(pQInfo); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query killed, abort", pQInfo); return; } @@ -998,7 +998,7 @@ static void vnodeSingleTableFixedOutputProcessor(SQInfo *pQInfo) { vnodeScanAllData(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } @@ -1033,7 +1033,7 @@ static void vnodeSingleTableMultiOutputProcessor(SQInfo *pQInfo) { vnodeScanAllData(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } @@ -1087,7 +1087,7 @@ static void vnodeSingleMeterIntervalMainLooper(STableQuerySupportObj *pSupporter initCtxOutputBuf(pRuntimeEnv); vnodeScanAllData(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } @@ -1301,7 +1301,7 @@ void vnodeSingleTableQuery(SSchedMsg *pMsg) { pQInfo->useconds += (taosGetTimestampUs() - st); /* check if query is killed or not */ - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed", pQInfo); pQInfo->over = 1; } else { @@ -1345,7 +1345,7 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { /* record the total elapsed time */ pQInfo->useconds += (taosGetTimestampUs() - st); - pQInfo->over = isQueryKilled(pQuery) ? 1 : 0; + pQInfo->over = isQueryKilled(pQInfo) ? 1 : 0; taosInterpoSetStartInfo(&pQInfo->pTableQuerySupporter->runtimeEnv.interpoInfo, pQuery->pointsRead, pQInfo->query.interpoType); diff --git a/src/vnode/detail/src/vnodeUtil.c b/src/vnode/detail/src/vnodeUtil.c index 43c24bae6af38676a7c9a4a3125f717dfc6d050c..1d5f45e5e351b0df01cef44c26e460acaca5d3c2 100644 --- a/src/vnode/detail/src/vnodeUtil.c +++ b/src/vnode/detail/src/vnodeUtil.c @@ -164,8 +164,7 @@ static int32_t vnodeBuildExprFromArithmeticStr(SSqlFunctionExpr* pExpr, SQueryMe SSchema* pSchema = toSchema(pQueryMsg, pColMsg, pQueryMsg->numOfCols); dTrace("qmsg:%p create binary expr from string:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz); - tSQLBinaryExprFromString(&pBinExpr, pSchema, pQueryMsg->numOfCols, pExpr->pBase.arg[0].argValue.pz, - pExpr->pBase.arg[0].argBytes); + tSQLBinaryExprFromString(&pBinExpr, pSchema, pQueryMsg->numOfCols, pExpr->pBase.arg[0].argValue.pz); if (pBinExpr == NULL) { dError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz); diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index 4964ac673f1b25d351d3eb5f0e8e146d510776db..ed6bb044c6c4f7c70f01a8ebedef9f034194df93 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -59,6 +59,8 @@ tsdb_repo_t * tsdbOpenRepo(char *tsdbDir); int32_t tsdbCloseRepo(tsdb_repo_t *repo); int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg); int32_t tsdbTriggerCommit(tsdb_repo_t *repo); +int32_t tsdbLockRepo(tsdb_repo_t *repo); +int32_t tsdbUnLockRepo(tsdb_repo_t *repo); // --------- TSDB TABLE DEFINITION typedef struct { @@ -88,15 +90,6 @@ int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg); int tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId); int tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg); -// Submit message for one table -typedef struct { - STableId tableId; - int32_t padding; // TODO just for padding here - int32_t sversion; // data schema version - int32_t len; // data part length, not including the SSubmitBlk head - char data[]; -} SSubmitBlk; - typedef struct { int32_t totalLen; int32_t len; @@ -106,15 +99,10 @@ typedef struct { int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); -// Submit message for this TSDB -typedef struct { - int32_t length; - int32_t compressed; - SSubmitBlk blocks[]; -} SSubmitMsg; - #define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) +struct STsdbRepo; + // SSubmitMsg Iterator typedef struct { int32_t totalLen; @@ -219,11 +207,6 @@ typedef struct SDataBlockInfo { int32_t sid; } SDataBlockInfo; -typedef struct STableIDList { - STableId *tableIds; - int32_t num; -} STableIDList; - typedef struct { } SFields; @@ -243,7 +226,7 @@ typedef void *tsdbpos_t; * @param pTableList table sid list * @return */ -tsdb_query_handle_t *tsdbQueryByTableId(STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo); +tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo); /** * move to next block @@ -337,15 +320,15 @@ tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stable * @param pQueryHandle * @return table sid list. the invoker is responsible for the release of this the sid list. */ -STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle); +SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle); /** - * Get the qualified table sid for a super table according to the tag query expression. + * Get the qualified table id for a super table according to the tag query expression. * @param stableid. super table sid * @param pTagCond. tag query condition * */ -STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond); +SArray *tsdbQueryTableList(struct STsdbRepo* tsdb, int64_t uid, const wchar_t *pTagCond, size_t len); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/inc/tsdbCache.h b/src/vnode/tsdb/inc/tsdbCache.h index 3bffa1c6a9ea75688e2e44b0f356a42570856d52..3e9eabc90d0eb2b177e7da19ef6af892f064c23d 100644 --- a/src/vnode/tsdb/inc/tsdbCache.h +++ b/src/vnode/tsdb/inc/tsdbCache.h @@ -17,7 +17,9 @@ #include +#include "taosdef.h" #include "tlist.h" +#include "tsdb.h" #ifdef __cplusplus extern "C" { @@ -38,18 +40,27 @@ typedef struct { SList * memPool; } STsdbCachePool; +typedef struct { + TSKEY keyFirst; + TSKEY keyLast; + int64_t numOfPoints; + SList * list; +} SCacheMem; + typedef struct { int maxBytes; int cacheBlockSize; + int totalCacheBlocks; STsdbCachePool pool; STsdbCacheBlock *curBlock; - SList * mem; - SList * imem; + SCacheMem * mem; + SCacheMem * imem; + tsdb_repo_t * pRepo; } STsdbCache; -STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize); +STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, tsdb_repo_t *pRepo); void tsdbFreeCache(STsdbCache *pCache); -void * tsdbAllocFromCache(STsdbCache *pCache, int bytes); +void * tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 89159a06e71af6c95c546c0b149c657026ff5c2e..6c42d4aa155addfd1daa8d9d98319dec9bff748d 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -17,12 +17,21 @@ #include +#include "dataformat.h" #include "taosdef.h" +#include "tglobalcfg.h" +#include "tsdb.h" #ifdef __cplusplus extern "C" { #endif +#define TSDB_FILE_HEAD_SIZE 512 +#define TSDB_FILE_DELIMITER 0xF00AFA0F + +#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) +#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) + typedef enum { TSDB_FILE_TYPE_HEAD = 0, // .head file type TSDB_FILE_TYPE_DATA, // .data file type @@ -30,20 +39,25 @@ typedef enum { TSDB_FILE_TYPE_MAX } TSDB_FILE_TYPE; +#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) < TSDB_FILE_TYPE_MAX) + extern const char *tsdbFileSuffix[]; typedef struct { - int64_t size; - int64_t tombSize; + int64_t size; // total size of the file + int64_t tombSize; // unused file size + int32_t totalBlocks; + int32_t totalSubBlocks; } SFileInfo; typedef struct { - int8_t type; + int fd; char fname[128]; - int64_t size; // total size of the file - int64_t tombSize; // unused file size + SFileInfo info; } SFile; +#define TSDB_IS_FILE_OPENED(f) ((f)->fd != -1) + typedef struct { int32_t fileId; SFile files[TSDB_FILE_TYPE_MAX]; @@ -51,21 +65,118 @@ typedef struct { // TSDB file handle typedef struct { - int32_t daysPerFile; - int32_t keep; - int32_t minRowPerFBlock; - int32_t maxRowsPerFBlock; - int32_t maxTables; + int maxFGroups; + int numOfFGroups; + SFileGroup fGroup[]; } STsdbFileH; -#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) < TSDB_FILE_TYPE_MAX) +#define TSDB_MIN_FILE_ID(fh) (fh)->fGroup[0].fileId +#define TSDB_MAX_FILE_ID(fh) (fh)->fGroup[(fh)->numOfFGroups - 1].fileId + +STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles); +void tsdbCloseFileH(STsdbFileH *pFileH); +int tsdbCreateFile(char *dataDir, int fileId, char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose); +int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); +int tsdbOpenFile(SFile *pFile, int oflag); +int tsdbCloseFile(SFile *pFile); SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); +int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid); + +#define TSDB_FGROUP_ITER_FORWARD 0 +#define TSDB_FGROUP_ITER_BACKWARD 1 +typedef struct { + int numOfFGroups; + SFileGroup *base; + SFileGroup *pFileGroup; + int direction; +} SFileGroupIter; + +void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction); +void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid); +SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter); + +typedef struct { + int32_t len; + int32_t offset; + int32_t hasLast : 1; + int32_t numOfSuperBlocks : 31; + int32_t checksum; + TSKEY maxKey; +} SCompIdx; /* sizeof(SCompIdx) = 24 */ + +/** + * if numOfSubBlocks == 0, then the SCompBlock is a sub-block + * if numOfSubBlocks >= 1, then the SCompBlock is a super-block + * - if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to + * the data block offset and length + * - if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the + * binary + */ +typedef struct { + int64_t last : 1; // If the block in data file or last file + int64_t offset : 63; // Offset of data block or sub-block index depending on numOfSubBlocks + int32_t algorithm : 8; // Compression algorithm + int32_t numOfPoints : 24; // Number of total points + int32_t sversion; // Schema version + int32_t len; // Data block length or nothing + int16_t numOfSubBlocks; // Number of sub-blocks; + int16_t numOfCols; + TSKEY keyFirst; + TSKEY keyLast; +} SCompBlock; + +#define IS_SUPER_BLOCK(pBlock) ((pBlock)->numOfSubBlocks >= 1) +#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0) + +typedef struct { + int32_t delimiter; // For recovery usage + int32_t checksum; // TODO: decide if checksum logic in this file or make it one API + int64_t uid; + SCompBlock blocks[]; +} SCompInfo; + +#define TSDB_COMPBLOCK_AT(pCompInfo, idx) ((pCompInfo)->blocks + (idx)) +#define TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pCompBlock, size)\ +do {\ + if (pCompBlock->numOfSubBlocks > 1) {\ + pCompBlock = pCompInfo->blocks + pCompBlock->offset;\ + size = pCompBlock->numOfSubBlocks;\ + } else {\ + size = 1;\ + }\ +} while (0) + +// TODO: take pre-calculation into account +typedef struct { + int16_t colId; // Column ID + int16_t len; // Column length + int32_t type : 8; + int32_t offset : 24; +} SCompCol; + +// TODO: Take recover into account +typedef struct { + int32_t delimiter; // For recovery usage + int32_t numOfCols; // For recovery usage + int64_t uid; // For recovery usage + SCompCol cols[]; +} SCompData; + +STsdbFileH* tsdbGetFile(tsdb_repo_t* pRepo); + +int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols); + +int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables); +int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf); +int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf); +int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf); +int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData); + +SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid); -STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32_t minRowsPerFBlock, - int32_t maxRowsPerFBlock, int32_t maxTables); +// TODO: need an API to merge all sub-block data into one -void tsdbCloseFile(STsdbFileH *pFileH); -int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup, int maxTables); +void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); #ifdef __cplusplus } #endif diff --git a/src/vnode/tsdb/inc/tsdbMeta.h b/src/vnode/tsdb/inc/tsdbMeta.h index 38f0818dfba826ee2300b6c0f2eee3fe2232423f..059ec87e914bbcfd416a3b160dfdeabe5e911065 100644 --- a/src/vnode/tsdb/inc/tsdbMeta.h +++ b/src/vnode/tsdb/inc/tsdbMeta.h @@ -33,20 +33,25 @@ extern "C" { #define IS_CREATE_STABLE(pCfg) ((pCfg)->tagValues != NULL) +typedef struct { + TSKEY keyFirst; + TSKEY keyLast; + int32_t numOfPoints; + void * pData; +} SMemTable; + // ---------- TSDB TABLE DEFINITION typedef struct STable { - int8_t type; - STableId tableId; - int32_t superUid; // Super table UID - int32_t sversion; - STSchema *schema; - STSchema *tagSchema; - SDataRow tagVal; - union { - void *pData; // For TSDB_NORMAL_TABLE and TSDB_CHILD_TABLE, it is the skiplist for cache data - void *pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index - } content; - void * iData; // Skiplist to commit + int8_t type; + STableId tableId; + int64_t superUid; // Super table UID + int32_t sversion; + STSchema * schema; + STSchema * tagSchema; + SDataRow tagVal; + SMemTable * mem; + SMemTable * imem; + void * pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index void * eventHandler; // TODO void * streamHandler; // TODO struct STable *next; // TODO: remove the next @@ -69,6 +74,8 @@ typedef struct { void *map; // table map of (uid ===> table) SMetaFile *mfh; // meta file handle + int maxRowBytes; + int maxCols; } STsdbMeta; STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables); @@ -91,11 +98,14 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta); #define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id] #define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */ +STsdbMeta* tsdbGetMeta(tsdb_repo_t* pRepo); + int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg); int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId); STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); -int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable); +// int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable); STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid); +char *getTupleKey(const void * data); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/vnode/tsdb/src/tsdbCache.c index 6a0741dced475965527ba61213032a00e3e197dd..d64b98d49dbcd7c7f76a7fe4c914a975c82c4658 100644 --- a/src/vnode/tsdb/src/tsdbCache.c +++ b/src/vnode/tsdb/src/tsdbCache.c @@ -14,12 +14,14 @@ */ #include +#include "tsdb.h" #include "tsdbCache.h" -static int tsdbAllocBlockFromPool(STsdbCache *pCache); +static int tsdbAllocBlockFromPool(STsdbCache *pCache); static void tsdbFreeBlockList(SList *list); +static void tsdbFreeCacheMem(SCacheMem *mem); -STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize) { +STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, tsdb_repo_t *pRepo) { STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache)); if (pCache == NULL) return NULL; @@ -27,9 +29,11 @@ STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize) { pCache->maxBytes = maxBytes; pCache->cacheBlockSize = cacheBlockSize; + pCache->pRepo = pRepo; int nBlocks = maxBytes / cacheBlockSize + 1; if (nBlocks <= 1) nBlocks = 2; + pCache->totalCacheBlocks = nBlocks; STsdbCachePool *pPool = &(pCache->pool); pPool->index = 0; @@ -46,11 +50,8 @@ STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize) { tdListAppend(pPool->memPool, (void *)(&pBlock)); } - pCache->mem = tdListNew(sizeof(STsdbCacheBlock *)); - if (pCache->mem == NULL) goto _err; - - pCache->imem = tdListNew(sizeof(STsdbCacheBlock *)); - if (pCache->imem == NULL) goto _err; + pCache->mem = NULL; + pCache->imem = NULL; return pCache; @@ -60,25 +61,24 @@ _err: } void tsdbFreeCache(STsdbCache *pCache) { - tsdbFreeBlockList(pCache->imem); - tsdbFreeBlockList(pCache->mem); + tsdbFreeCacheMem(pCache->imem); + tsdbFreeCacheMem(pCache->mem); tsdbFreeBlockList(pCache->pool.memPool); free(pCache); } -void *tsdbAllocFromCache(STsdbCache *pCache, int bytes) { +void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) { if (pCache == NULL) return NULL; if (bytes > pCache->cacheBlockSize) return NULL; - if (isListEmpty(pCache->mem)) { - if (tsdbAllocBlockFromPool(pCache) < 0) { - // TODO: deal with the error + if (pCache->curBlock == NULL || pCache->curBlock->remain < bytes) { + if (pCache->curBlock !=NULL && (pCache->mem->list) >= pCache->totalCacheBlocks/2) { + tsdbTriggerCommit(pCache->pRepo); } - } - if (pCache->curBlock->remain < bytes) { - if (tsdbAllocBlockFromPool(pCache) < 0) { + while (tsdbAllocBlockFromPool(pCache) < 0) { // TODO: deal with the error + // printf("Failed to allocate from cache pool\n"); } } @@ -86,12 +86,14 @@ void *tsdbAllocFromCache(STsdbCache *pCache, int bytes) { pCache->curBlock->offset += bytes; pCache->curBlock->remain -= bytes; memset(ptr, 0, bytes); + if (key < pCache->mem->keyFirst) pCache->mem->keyFirst = key; + if (key > pCache->mem->keyLast) pCache->mem->keyLast = key; + pCache->mem->numOfPoints++; return ptr; } static void tsdbFreeBlockList(SList *list) { - if (list == NULL) return; SListNode * node = NULL; STsdbCacheBlock *pBlock = NULL; while ((node = tdListPopHead(list)) != NULL) { @@ -102,9 +104,21 @@ static void tsdbFreeBlockList(SList *list) { tdListFree(list); } +static void tsdbFreeCacheMem(SCacheMem *mem) { + if (mem == NULL) return; + SList *list = mem->list; + tsdbFreeBlockList(list); + free(mem); +} + static int tsdbAllocBlockFromPool(STsdbCache *pCache) { STsdbCachePool *pPool = &(pCache->pool); - if (listNEles(pPool->memPool) == 0) return -1; + + tsdbLockRepo(pCache->pRepo); + if (listNEles(pPool->memPool) == 0) { + tsdbUnLockRepo(pCache->pRepo); + return -1; + } SListNode *node = tdListPopHead(pPool->memPool); @@ -114,8 +128,19 @@ static int tsdbAllocBlockFromPool(STsdbCache *pCache) { pBlock->offset = 0; pBlock->remain = pCache->cacheBlockSize; - tdListAppendNode(pCache->mem, node); + if (pCache->mem == NULL) { // Create a new one + pCache->mem = (SCacheMem *)malloc(sizeof(SCacheMem)); + if (pCache->mem == NULL) return NULL; + pCache->mem->keyFirst = INT64_MAX; + pCache->mem->keyLast = 0; + pCache->mem->numOfPoints = 0; + pCache->mem->list = tdListNew(sizeof(STsdbCacheBlock *)); + } + + tdListAppendNode(pCache->mem->list, node); pCache->curBlock = pBlock; + tsdbUnLockRepo(pCache->pRepo); + return 0; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 8a7e40cabd0d7e864c9190009cd4a52deaf436cd..d6964112e7fef975aaf0b34c67faefe268fdab4e 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -22,214 +22,374 @@ #include #include -#include "tglobalcfg.h" +#include "tutil.h" #include "tsdbFile.h" -#define TSDB_FILE_HEAD_SIZE 512 -#define TSDB_FILE_DELIMITER 0xF00AFA0F - -#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) -#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) - -typedef struct { - int32_t len; - int32_t padding; // For padding purpose - int64_t offset; -} SCompIdx; - -/** - * if numOfSubBlocks == -1, then the SCompBlock is a sub-block - * if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to - * the data block offset and length - * if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the - * binary - */ -typedef struct { - int64_t last : 1; // If the block in data file or last file - int64_t offset : 63; // Offset of data block or sub-block index depending on numOfSubBlocks - int32_t algorithm : 8; // Compression algorithm - int32_t numOfPoints : 24; // Number of total points - int32_t sversion; // Schema version - int32_t len; // Data block length or nothing - int16_t numOfSubBlocks; // Number of sub-blocks; - int16_t numOfCols; - TSKEY keyFirst; - TSKEY keyLast; -} SCompBlock; - -typedef struct { - int32_t delimiter; // For recovery usage - int32_t checksum; // TODO: decide if checksum logic in this file or make it one API - int64_t uid; - int32_t padding; // For padding purpose - int32_t numOfBlocks; // TODO: make the struct padding - SCompBlock blocks[]; -} SCompInfo; - -// TODO: take pre-calculation into account -typedef struct { - int16_t colId; // Column ID - int16_t len; // Column length - int32_t type : 8; - int32_t offset : 24; -} SCompCol; - -// TODO: Take recover into account -typedef struct { - int32_t delimiter; // For recovery usage - int32_t numOfCols; // For recovery usage - int64_t uid; // For recovery usage - SCompCol cols[]; -} SCompData; - const char *tsdbFileSuffix[] = { ".head", // TSDB_FILE_TYPE_HEAD ".data", // TSDB_FILE_TYPE_DATA ".last" // TSDB_FILE_TYPE_LAST }; -static int tsdbWriteFileHead(int fd, SFile *pFile) { - char head[TSDB_FILE_HEAD_SIZE] = "\0"; +static int compFGroupKey(const void *key, const void *fgroup); +static int compFGroup(const void *arg1, const void *arg2); +static int tsdbGetFileName(char *dataDir, int fileId, char *suffix, char *fname); +static int tsdbWriteFileHead(SFile *pFile); +static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables); +static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid); - pFile->size += TSDB_FILE_HEAD_SIZE; +STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { + STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(STsdbFileH) + sizeof(SFileGroup) * maxFiles); + if (pFileH == NULL) { // TODO: deal with ERROR here + return NULL; + } - // TODO: write version and File statistic to the head - lseek(fd, 0, SEEK_SET); - if (write(fd, head, TSDB_FILE_HEAD_SIZE) < 0) return -1; + pFileH->maxFGroups = maxFiles; + DIR *dir = opendir(dataDir); + if (dir == NULL) { + free(pFileH); + return NULL; + } + + struct dirent *dp = NULL; + int fid = 0; + SFileGroup fGroup = {0}; + while ((dp = readdir(dir)) != NULL) { + if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 1) == 0) continue; + int fid = 0; + sscanf(dp->d_name, "f%d", &fid); + if (tsdbOpenFGroup(pFileH, dataDir, fid) < 0) { + break; + // TODO + } + } + + return pFileH; +} + +void tsdbCloseFileH(STsdbFileH *pFileH) { free(pFileH); } + +static int tsdbInitFile(char *dataDir, int fid, char *suffix, SFile *pFile) { + tsdbGetFileName(dataDir, fid, suffix, pFile->fname); + if (access(pFile->fname, F_OK|R_OK|W_OK) < 0) return -1; + pFile->fd = -1; + // TODO: recover the file info + // pFile->info = {0}; return 0; } -static int tsdbWriteHeadFileIdx(int fd, int maxTables, SFile *pFile) { - int size = sizeof(SCompIdx) * maxTables; - void *buf = calloc(1, size); - if (buf == NULL) return -1; +static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) { + if (tsdbSearchFGroup(pFileH, fid) != NULL) return 0; - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { - free(buf); - return -1; - } + char fname[128] = "\0"; + SFileGroup fGroup = {0}; + fGroup.fileId = fid; - if (write(fd, buf, size) < 0) { - free(buf); - return -1; + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + if (tsdbInitFile(dataDir, fid, tsdbFileSuffix[type], &fGroup.files[type]) < 0) return -1; } + pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; + qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); + return 0; +} - pFile->size += size; +int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) { + if (pFileH->numOfFGroups >= pFileH->maxFGroups) return -1; + + SFileGroup fGroup; + SFileGroup *pFGroup = &fGroup; + if (tsdbSearchFGroup(pFileH, fid) == NULL) { // if not exists, create one + pFGroup->fileId = fid; + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]), type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0) { + // TODO: deal with the ERROR here, remove those creaed file + return -1; + } + } + pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; + qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); + } return 0; } -static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname) { - if (dataDir == NULL || fname == NULL || !IS_VALID_TSDB_FILE_TYPE(type)) return -1; +int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { + SFileGroup *pGroup = + bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey); + if (pGroup == NULL) return -1; - sprintf(fname, "%s/f%d%s", dataDir, fileId, tsdbFileSuffix[type]); + // Remove from disk + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + remove(pGroup->files[type].fname); + } + + // Adjust the memory + int filesBehind = pFileH->numOfFGroups - (((char *)pGroup - (char *)(pFileH->fGroup)) / sizeof(SFileGroup) + 1); + if (filesBehind > 0) { + memmove((void *)pGroup, (void *)((char *)pGroup + sizeof(SFileGroup)), sizeof(SFileGroup) * filesBehind); + } + pFileH->numOfFGroups--; return 0; } -/** - * Create a file and set the SFile object - */ -static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, SFile *pFile) { - memset((void *)pFile, 0, sizeof(SFile)); - pFile->type = type; +void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { + pIter->direction = direction; + pIter->base = pFileH->fGroup; + pIter->numOfFGroups = pFileH->numOfFGroups; + if (pFileH->numOfFGroups == 0){ + pIter->pFileGroup = NULL; + } else { + if (direction == TSDB_FGROUP_ITER_FORWARD) { + pIter->pFileGroup = pFileH->fGroup; + } else { + pIter->pFileGroup = pFileH->fGroup + pFileH->numOfFGroups - 1; + } + } +} - tsdbGetFileName(dataDir, fileId, type, pFile->fname); - if (access(pFile->fname, F_OK) == 0) { - // File already exists - return -1; +void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { + int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; + void *ptr = taosbsearch(&fid, pIter->base, sizeof(SFileGroup), pIter->numOfFGroups, compFGroupKey, flags); + if (ptr == NULL) { + pIter->pFileGroup = NULL; + } else { + pIter->pFileGroup = (SFileGroup *)ptr; } +} - int fd = open(pFile->fname, O_WRONLY | O_CREAT, 0755); - if (fd < 0) return -1; +SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { + SFileGroup *ret = pIter->pFileGroup; + if (ret == NULL) return NULL; - if (type == TSDB_FILE_TYPE_HEAD) { - if (tsdbWriteHeadFileIdx(fd, maxTables, pFile) < 0) { - close(fd); - return -1; + if (pIter->direction = TSDB_FGROUP_ITER_FORWARD) { + if (pIter->pFileGroup + 1 == pIter->base + pIter->numOfFGroups) { + pIter->pFileGroup = NULL; + } else { + pIter->pFileGroup += 1; + } + } else { + if (pIter->pFileGroup - 1 == pIter->base) { + pIter->pFileGroup = NULL; + } else { + pIter->pFileGroup -= 1; } } + return ret; +} - if (tsdbWriteFileHead(fd, pFile) < 0) { - close(fd); - return -1; +int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) { + SCompBlock *pBlock = pStartBlock; + for (int i = 0; i < numOfBlocks; i++) { + if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1; + for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) { + SCompCol *pCompCol = &(pCompData->cols[iCol]); + pCols->numOfPoints += pBlock->numOfPoints; + int k = 0; + for (; k < pCols->numOfCols; k++) { + if (pCompCol->colId == pCols->cols[k].colId) break; + } + + if (tsdbLoadColData(pFile, pCompCol, pBlock->offset, + (void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0) + return -1; + } + pStartBlock++; } + return 0; +} + +int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols) { + SCompBlock *pSuperBlock = TSDB_COMPBLOCK_AT(pCompInfo, idx); + SCompBlock *pStartBlock = NULL; + SCompBlock *pBlock = NULL; + int numOfBlocks = pSuperBlock->numOfSubBlocks; + + if (numOfBlocks == 1) + pStartBlock = pSuperBlock; + else + pStartBlock = TSDB_COMPBLOCK_AT(pCompInfo, pSuperBlock->offset); + + int maxNumOfCols = 0; + pBlock = pStartBlock; + for (int i = 0; i < numOfBlocks; i++) { + if (pBlock->numOfCols > maxNumOfCols) maxNumOfCols = pBlock->numOfCols; + pBlock++; + } + + SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * maxNumOfCols); + if (pCompData == NULL) return -1; - close(fd); + // Load data from the block + if (tsdbLoadDataBlock(pOutFile, pStartBlock, numOfBlocks, pCols, pCompData)); + // Write data block to the file + { + // TODO + } + + + if (pCompData) free(pCompData); return 0; } -static int tsdbRemoveFile(SFile *pFile) { - if (pFile == NULL) return -1; - return remove(pFile->fname); +int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) { + SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); + if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + + if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1; + // TODO: need to check the correctness + return 0; } -// Create a file group with fileId and return a SFileGroup object -int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup, int maxTables) { - if (dataDir == NULL || pFGroup == NULL) return -1; +int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) { + SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); - memset((void *)pFGroup, 0, sizeof(SFileGroup)); + if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(dataDir, fileId, type, maxTables, &(pFGroup->files[type])) < 0) { - // TODO: deal with the error here, remove the created files - return -1; - } - } + if (read(pFile->fd, buf, pIdx->len) < 0) return -1; - pFGroup->fileId = fileId; + // TODO: need to check the correctness return 0; } -/** - * Initialize the TSDB file handle - */ -STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32_t minRowsPerFBlock, - int32_t maxRowsPerFBlock, int32_t maxTables) { - STsdbFileH *pTsdbFileH = - (STsdbFileH *)calloc(1, sizeof(STsdbFileH) + sizeof(SFileGroup) * tsdbGetMaxNumOfFiles(keep, daysPerFile)); - if (pTsdbFileH == NULL) return NULL; - - pTsdbFileH->daysPerFile = daysPerFile; - pTsdbFileH->keep = keep; - pTsdbFileH->minRowPerFBlock = minRowsPerFBlock; - pTsdbFileH->maxRowsPerFBlock = maxRowsPerFBlock; - pTsdbFileH->maxTables = maxTables; - - // Open the directory to read information of each file - DIR *dir = opendir(dataDir); - if (dir == NULL) { - free(pTsdbFileH); - return NULL; +int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf) { + // assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); + + if (lseek(pFile->fd, pBlock->offset, SEEK_SET) < 0) return -1; + size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols; + if (read(pFile->fd, buf, size) < 0) return -1; + + return 0; +} + +int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf) { + if (lseek(pFile->fd, blockBaseOffset + pCol->offset, SEEK_SET) < 0) return -1; + if (read(pFile->fd, buf, pCol->len) < 0) return -1; + return 0; +} + +static int compFGroupKey(const void *key, const void *fgroup) { + int fid = *(int *)key; + SFileGroup *pFGroup = (SFileGroup *)fgroup; + return (fid - pFGroup->fileId); +} + +static int compFGroup(const void *arg1, const void *arg2) { + return ((SFileGroup *)arg1)->fileId - ((SFileGroup *)arg2)->fileId; +} + +static int tsdbWriteFileHead(SFile *pFile) { + char head[TSDB_FILE_HEAD_SIZE] = "\0"; + + pFile->info.size += TSDB_FILE_HEAD_SIZE; + + // TODO: write version and File statistic to the head + lseek(pFile->fd, 0, SEEK_SET); + if (write(pFile->fd, head, TSDB_FILE_HEAD_SIZE) < 0) return -1; + + return 0; +} + +static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) { + int size = sizeof(SCompIdx) * maxTables; + void *buf = calloc(1, size); + if (buf == NULL) return -1; + + if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + free(buf); + return -1; } - char fname[256]; + if (write(pFile->fd, buf, size) < 0) { + free(buf); + return -1; + } - struct dirent *dp; - while ((dp = readdir(dir)) != NULL) { - if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue; - if (true /* check if the file is the .head file */) { - int fileId = 0; - int vgId = 0; - sscanf(dp->d_name, "v%df%d.head", &vgId, &fileId); - // TODO + pFile->info.size += size; + + free(buf); + return 0; +} - // Open head file +static int tsdbGetFileName(char *dataDir, int fileId, char *suffix, char *fname) { + if (dataDir == NULL || fname == NULL) return -1; - // Open data file + sprintf(fname, "%s/f%d%s", dataDir, fileId, suffix); - // Open last file + return 0; +} + +int tsdbOpenFile(SFile *pFile, int oflag) { // TODO: change the function + if (TSDB_IS_FILE_OPENED(pFile)) return -1; + + pFile->fd = open(pFile->fname, oflag, 0755); + if (pFile->fd < 0) return -1; + + return 0; +} + +int tsdbCloseFile(SFile *pFile) { + int ret = close(pFile->fd); + pFile->fd = -1; + return ret; +} + +SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) { + SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid); + if (pGroup == NULL) return NULL; + + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + tsdbOpenFile(&(pGroup->files[type]), O_RDWR); + } + return pGroup; +} + +int tsdbCreateFile(char *dataDir, int fileId, char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose) { + memset((void *)pFile, 0, sizeof(SFile)); + pFile->fd = -1; + + tsdbGetFileName(dataDir, fileId, suffix, pFile->fname); + + if (access(pFile->fname, F_OK) == 0) { + // File already exists + return -1; + } + + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) { + // TODO: deal with the ERROR here + return -1; + } + + if (writeHeader) { + if (tsdbWriteHeadFileIdx(pFile, maxTables) < 0) { + tsdbCloseFile(pFile); + return -1; } } - return pTsdbFileH; + if (tsdbWriteFileHead(pFile) < 0) { + tsdbCloseFile(pFile); + return -1; + } + + if (toClose) tsdbCloseFile(pFile); + + return 0; } -static void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, +void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; +} + +SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid) { + if (pFileH->numOfFGroups == 0 || fid < pFileH->fGroup[0].fileId || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) + return NULL; + void *ptr = bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey); + if (ptr == NULL) return NULL; + return (SFileGroup *)ptr; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index ed95eac5bc9fc8c41b78fb2944932d4909415ab2..6d675d300a418b521fd5595396c25f3dfac88d55 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -8,6 +8,7 @@ #include #include #include +#include #include // #include "taosdef.h" @@ -44,6 +45,8 @@ #define TSDB_CFG_FILE_NAME "CONFIG" #define TSDB_DATA_DIR_NAME "data" +#define TSDB_DEFAULT_FILE_BLOCK_ROW_OPTION 0.7 +#define TSDB_MAX_LAST_FILE_SIZE (1024 * 1024 * 10) // 10M enum { TSDB_REPO_STATE_ACTIVE, TSDB_REPO_STATE_CLOSED, TSDB_REPO_STATE_CONFIGURING }; @@ -83,7 +86,12 @@ static int tsdbOpenMetaFile(char *tsdbDir); static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); -static void * tsdbCommitToFile(void *arg); +static void * tsdbCommitData(void *arg); +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols); +static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey); +static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey); +static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, + int64_t uid); #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid] #define TSDB_GET_TABLE_BY_NAME(pRepo, name) @@ -149,6 +157,7 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO pRepo->rootDir = strdup(rootDir); pRepo->config = *pCfg; pRepo->limiter = limiter; + pthread_mutex_init(&pRepo->mutex, NULL); // Create the environment files and directories if (tsdbSetRepoEnv(pRepo) < 0) { @@ -167,7 +176,7 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO pRepo->tsdbMeta = pMeta; // Initialize cache - STsdbCache *pCache = tsdbInitCache(pCfg->maxCacheSize, -1); + STsdbCache *pCache = tsdbInitCache(pCfg->maxCacheSize, -1, (tsdb_repo_t *)pRepo); if (pCache == NULL) { free(pRepo->rootDir); tsdbFreeMeta(pRepo->tsdbMeta); @@ -180,7 +189,7 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO char dataDir[128] = "\0"; tsdbGetDataDirName(pRepo, dataDir); pRepo->tsdbFileH = - tsdbInitFile(dataDir, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->maxTables); + tsdbInitFileH(dataDir, pCfg->maxTables); if (pRepo->tsdbFileH == NULL) { free(pRepo->rootDir); tsdbFreeCache(pRepo->tsdbCache); @@ -228,6 +237,7 @@ int32_t tsdbDropRepo(tsdb_repo_t *repo) { * @return a TSDB repository handle on success, NULL for failure and the error number is set */ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) { + char dataDir[128] = "\0"; if (access(tsdbDir, F_OK | W_OK | R_OK) < 0) { return NULL; } @@ -248,7 +258,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) { return NULL; } - pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1); + pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (tsdb_repo_t *)pRepo); if (pRepo->tsdbCache == NULL) { tsdbFreeMeta(pRepo->tsdbMeta); free(pRepo->rootDir); @@ -256,6 +266,16 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) { return NULL; } + tsdbGetDataDirName(pRepo, dataDir); + pRepo->tsdbFileH = tsdbInitFileH(dataDir, pRepo->config.maxTables); + if (pRepo->tsdbFileH == NULL) { + tsdbFreeCache(pRepo->tsdbCache); + tsdbFreeMeta(pRepo->tsdbMeta); + free(pRepo->rootDir); + free(pRepo); + return NULL; + } + pRepo->state = TSDB_REPO_STATE_ACTIVE; return (tsdb_repo_t *)pRepo; @@ -278,8 +298,29 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo) { if (pRepo == NULL) return 0; pRepo->state = TSDB_REPO_STATE_CLOSED; + tsdbLockRepo(repo); + if (pRepo->commit) { + tsdbUnLockRepo(repo); + return -1; + } + pRepo->commit = 1; + // Loop to move pData to iData + for (int i = 0; i < pRepo->config.maxTables; i++) { + STable *pTable = pRepo->tsdbMeta->tables[i]; + if (pTable != NULL && pTable->mem != NULL) { + pTable->imem = pTable->mem; + pTable->mem = NULL; + } + } + // TODO: Loop to move mem to imem + pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; + pRepo->tsdbCache->mem = NULL; + pRepo->tsdbCache->curBlock = NULL; + tsdbUnLockRepo(repo); - tsdbFlushCache(pRepo); + tsdbCommitData((void *)repo); + + tsdbCloseFileH(pRepo->tsdbFileH); tsdbFreeMeta(pRepo->tsdbMeta); @@ -304,28 +345,44 @@ int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) { int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; - - if (pthread_mutex_lock(&(pRepo->mutex)) < 0) return -1; - if (pRepo->commit) return 0; + + tsdbLockRepo(repo); + if (pRepo->commit) { + tsdbUnLockRepo(repo); + return -1; + } pRepo->commit = 1; // Loop to move pData to iData for (int i = 0; i < pRepo->config.maxTables; i++) { STable *pTable = pRepo->tsdbMeta->tables[i]; - if (pTable != NULL) { - void *pData = pTable->content.pData; - pTable->content.pData = NULL; - pTable->iData = pData; + if (pTable != NULL && pTable->mem != NULL) { + pTable->imem = pTable->mem; + pTable->mem = NULL; } } - // Loop to move mem to imem - tdListMove(pRepo->tsdbCache->mem, pRepo->tsdbCache->imem); + // TODO: Loop to move mem to imem + pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; + pRepo->tsdbCache->mem = NULL; + pRepo->tsdbCache->curBlock = NULL; + tsdbUnLockRepo(repo); + + // TODO: here should set as detached or use join for memory leak + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); + pthread_create(&(pRepo->commitThread), &thattr, tsdbCommitData, (void *)repo); - pthread_create(&(pRepo->commitThread), NULL, tsdbCommitToFile, (void *)repo); - pthread_mutex_unlock(&(pRepo->mutex)); + return 0; +} - pthread_join(pRepo->commitThread, NULL); +int32_t tsdbLockRepo(tsdb_repo_t *repo) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + return pthread_mutex_lock(&(pRepo->mutex)); +} - return 0; +int32_t tsdbUnLockRepo(tsdb_repo_t *repo) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + return pthread_mutex_unlock(&(pRepo->mutex)); } /** @@ -484,6 +541,10 @@ SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL || pIter == NULL) return -1; + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + pMsg->compressed = htonl(pMsg->compressed); + pIter->totalLen = pMsg->length; pIter->len = TSDB_SUBMIT_MSG_HEAD_SIZE; if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) { @@ -498,7 +559,15 @@ int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) { SSubmitBlk *pBlock = pIter->pBlock; if (pBlock == NULL) return NULL; - + + pBlock->len = htonl(pBlock->len); + pBlock->numOfRows = htons(pBlock->numOfRows); + pBlock->uid = htobe64(pBlock->uid); + pBlock->tid = htonl(pBlock->tid); + + pBlock->sversion = htonl(pBlock->sversion); + pBlock->padding = htonl(pBlock->padding); + pIter->len = pIter->len + sizeof(SSubmitBlk) + pBlock->len; if (pIter->len >= pIter->totalLen) { pIter->pBlock = NULL; @@ -509,6 +578,16 @@ SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) { return pBlock; } +STsdbMeta* tsdbGetMeta(tsdb_repo_t* pRepo) { + STsdbRepo *tsdb = (STsdbRepo *)pRepo; + return tsdb->tsdbMeta; +} + +STsdbFileH* tsdbGetFile(tsdb_repo_t* pRepo) { + STsdbRepo* tsdb = (STsdbRepo*) pRepo; + return tsdb->tsdbFileH; +} + // Check the configuration and set default options static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { // Check precision @@ -669,10 +748,21 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable int32_t level = 0; int32_t headSize = 0; - tSkipListRandNodeInfo(pTable->content.pData, &level, &headSize); + if (pTable->mem == NULL) { + pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); + if (pTable->mem == NULL) return -1; + pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey); + pTable->mem->keyFirst = INT64_MAX; + pTable->mem->keyLast = 0; + } + + tSkipListRandNodeInfo(pTable->mem->pData, &level, &headSize); + TSKEY key = dataRowKey(row); + // printf("insert:%lld, size:%d\n", key, pTable->mem->numOfPoints); + // Copy row into the memory - SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row)); + SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row), key); if (pNode == NULL) { // TODO: deal with allocate failure } @@ -681,7 +771,19 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable dataRowCpy(SL_GET_NODE_DATA(pNode), row); // Insert the skiplist node into the data - tsdbInsertRowToTableImpl(pNode, pTable); + if (pTable->mem == NULL) { + pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); + if (pTable->mem == NULL) return -1; + pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey); + pTable->mem->keyFirst = INT64_MAX; + pTable->mem->keyLast = 0; + } + tSkipListPut(pTable->mem->pData, pNode); + if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; + if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; + + pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData); +// pTable->mem->numOfPoints++; return 0; } @@ -689,7 +791,8 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { STsdbRepo *pRepo = (STsdbRepo *)repo; - STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, pBlock->tableId); + STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid}; + STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId); if (pTable == NULL) return -1; SSubmitBlkIter blkIter; @@ -705,21 +808,458 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { return 0; } -static void *tsdbCommitToFile(void *arg) { +static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { + int numOfRows = 0; + + do { + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) break; + + SDataRow row = SL_GET_NODE_DATA(node); + if (dataRowKey(row) > maxKey) break; + + tdAppendDataRowToDataCol(row, pCols); + + numOfRows++; + if (numOfRows >= maxRowsToRead) break; + } while (tSkipListIterNext(pIter)); + + return numOfRows; +} + +static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) { + if (iters == NULL) return; + + for (int tid = 0; tid < maxTables; tid++) { + if (iters[tid] == NULL) continue; + tSkipListDestroyIter(iters[tid]); + } + + free(iters); +} + +static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) { + SSkipListIterator **iters = (SSkipListIterator *)calloc(maxTables, sizeof(SSkipListIterator *)); + if (iters == NULL) return NULL; + + for (int tid = 0; tid < maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + if (pTable == NULL || pTable->imem == NULL) continue; + + iters[tid] = tSkipListCreateIter(pTable->imem->pData); + if (iters[tid] == NULL) { + tsdbDestroyTableIters(iters, maxTables); + return NULL; + } + + if (!tSkipListIterNext(iters[tid])) { + // No data in this iterator + tSkipListDestroyIter(iters[tid]); + iters[tid] = NULL; + } + } + + return iters; +} + +// Commit to file +static void *tsdbCommitData(void *arg) { // TODO - STsdbRepo *pRepo = (STsdbRepo *)arg; - STsdbMeta *pMeta = pRepo->tsdbMeta; - for (int i = 0; i < pRepo->config.maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - SSkipListIterator *pIter = tSkipListCreateIter(pTable->iData); - while (tSkipListIterNext(pIter)) { - SSkipListNode *node = tSkipListIterGet(pIter); - SDataRow row = SL_GET_NODE_DATA(node); - int k = 0; + printf("Starting to commit....\n"); + STsdbRepo * pRepo = (STsdbRepo *)arg; + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbCache *pCache = pRepo->tsdbCache; + STsdbCfg * pCfg = &(pRepo->config); + if (pCache->imem == NULL) return NULL; + + // Create the iterator to read from cache + SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables); + if (iters == NULL) { + // TODO: deal with the error + return NULL; + } + // Create a data column buffer for commit + SDataCols *pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock); + if (pDataCols == NULL) { + // TODO: deal with the error + return NULL; + } + + int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); + int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); + + for (int fid = sfid; fid <= efid; fid++) { + if (tsdbCommitToFile(pRepo, fid, iters, pDataCols) < 0) { + // TODO: deal with the error here + // assert(0); + } + } + + tdFreeDataCols(pDataCols); + tsdbDestroyTableIters(iters, pCfg->maxTables); + + tsdbLockRepo(arg); + tdListMove(pCache->imem->list, pCache->pool.memPool); + free(pCache->imem); + pCache->imem = NULL; + pRepo->commit = 0; + // TODO: free the skiplist + for (int i = 0; i < pCfg->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->imem) { // Here has memory leak + pTable->imem = NULL; } } + tsdbUnLockRepo(arg); return NULL; +} + +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) { + int isNewLastFile = 0; + + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + STsdbCfg * pCfg = &pRepo->config; + SFile hFile, lFile; + SFileGroup *pGroup = NULL; + SCompIdx * pIndices = NULL; + SCompInfo * pCompInfo = NULL; + size_t compInfoSize = 0; + SCompBlock compBlock; + SCompBlock *pBlock = &compBlock; + + TSKEY minKey = 0, maxKey = 0; + tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + + // Check if there are data to commit to this file + int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); + if (!hasDataToCommit) return 0; // No data to commit, just return + + // TODO: make it more flexible + pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + sizeof(SCompBlock) * 1000); + + // Create and open files for commit + tsdbGetDataDirName(pRepo, dataDir); + if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { /* TODO */ + } + pGroup = tsdbOpenFilesForCommit(pFileH, fid); + if (pGroup == NULL) { /* TODO */ + } + tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &hFile, 1, 0); + if (0 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { + // TODO: make it not to write the last file every time + tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0); + isNewLastFile = 1; + } + + // Load the SCompIdx + pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); + if (pIndices == NULL) { /* TODO*/ + } + if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { /* TODO */ + } + + lseek(hFile.fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, SEEK_SET); + + // Loop to commit data in each table + for (int tid = 0; tid < pCfg->maxTables; tid++) { + STable * pTable = pMeta->tables[tid]; + SSkipListIterator *pIter = iters[tid]; + SCompIdx * pIdx = &pIndices[tid]; + + int nNewBlocks = 0; + + if (pTable == NULL || pIter == NULL) continue; + + /* If no new data to write for this table, just write the old data to new file + * if there are. + */ + if (!tsdbHasDataInRange(pIter, minKey, maxKey)) { + // has old data + if (pIdx->len > 0) { + goto _table_over; + // if (isNewLastFile && pIdx->hasLast) { + if (0) { + // need to move the last block to new file + if ((pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len)) == NULL) { /* TODO */ + } + if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */ + } + + tdInitDataCols(pCols, pTable->schema); + + SCompBlock *pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks); + int nBlocks = 0; + + TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pTBlock, nBlocks); + + SCompBlock tBlock; + int64_t toffset, tlen; + tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_LAST], pTBlock, nBlocks, pCols, &tBlock); + + tsdbWriteBlockToFileImpl(&lFile, pCols, pCols->numOfPoints, &toffset, tlen, pTable->tableId.uid); + pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks); + pTBlock->offset = toffset; + pTBlock->len = tlen; + pTBlock->numOfPoints = pCols->numOfPoints; + pTBlock->numOfSubBlocks = 1; + + pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); + if (nBlocks > 1) { + pIdx->len -= (sizeof(SCompBlock) * nBlocks); + } + write(hFile.fd, (void *)pCompInfo, pIdx->len); + } else { + pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); + sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len); + hFile.info.size += pIdx->len; + } + } + continue; + } + + pCompInfo->delimiter = TSDB_FILE_DELIMITER; + pCompInfo->checksum = 0; + pCompInfo->uid = pTable->tableId.uid; + + // Load SCompBlock part if neccessary + int isCompBlockLoaded = 0; + if (0) { + // if (pIdx->offset > 0) { + if (pIdx->hasLast || tsdbHasDataInRange(pIter, minKey, pIdx->maxKey)) { + // has last block || cache key overlap with commit key + pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len + sizeof(SCompBlock) * 100); + if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */ + } + if (pCompInfo->uid == pTable->tableId.uid) isCompBlockLoaded = 1; + } else { + // TODO: No need to load the SCompBlock part, just sendfile the SCompBlock part + // and write those new blocks to it + } + } + + tdInitDataCols(pCols, pTable->schema); + + int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; + while (1) { + tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols); + if (pCols->numOfPoints == 0) break; + + int pointsWritten = pCols->numOfPoints; + // TODO: all write to the end of .data file + int64_t toffset = 0; + int32_t tlen = 0; + tsdbWriteBlockToFileImpl(&pGroup->files[TSDB_FILE_TYPE_DATA], pCols, pCols->numOfPoints, &toffset, &tlen, pTable->tableId.uid); + + // Make the compBlock + SCompBlock *pTBlock = pCompInfo->blocks + nNewBlocks++; + pTBlock->offset = toffset; + pTBlock->len = tlen; + pTBlock->keyFirst = dataColsKeyFirst(pCols); + pTBlock->keyLast = dataColsKeyLast(pCols); + pTBlock->last = 0; + pTBlock->algorithm = 0; + pTBlock->numOfPoints = pCols->numOfPoints; + pTBlock->sversion = pTable->sversion; + pTBlock->numOfSubBlocks = 1; + + if (dataColsKeyLast(pCols) > pIdx->maxKey) pIdx->maxKey = dataColsKeyLast(pCols); + + tdPopDataColsPoints(pCols, pointsWritten); + maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pCols->numOfPoints; + } + + +_table_over: + // Write the SCompBlock part + pIdx->offset = lseek(hFile.fd, 0, SEEK_END); + if (pIdx->len > 0) { + sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len); + if (nNewBlocks > 0) { + write(hFile.fd, (void *)(pCompInfo->blocks), sizeof(SCompBlock) * nNewBlocks); + pIdx->len += (sizeof(SCompBlock) * nNewBlocks); + } + } else { + if (nNewBlocks > 0) { + write(hFile.fd, (void *)pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks); + pIdx->len += sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks; + } + } + + pIdx->checksum = 0; + pIdx->numOfSuperBlocks += nNewBlocks; + pIdx->hasLast = 0; + } + + // Write the SCompIdx part + if (lseek(hFile.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {/* TODO */} + if (write(hFile.fd, (void *)pIndices, sizeof(SCompIdx) * pCfg->maxTables) < 0) {/* TODO */} + + // close the files + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + tsdbCloseFile(&pGroup->files[type]); + } + tsdbCloseFile(&hFile); + if (isNewLastFile) tsdbCloseFile(&lFile); + // TODO: replace the .head and .last file + rename(hFile.fname, pGroup->files[TSDB_FILE_TYPE_HEAD].fname); + pGroup->files[TSDB_FILE_TYPE_HEAD].info = hFile.info; + if (isNewLastFile) { + rename(lFile.fname, pGroup->files[TSDB_FILE_TYPE_LAST].fname); + pGroup->files[TSDB_FILE_TYPE_LAST].info = lFile.info; + } + + if (pIndices) free(pIndices); + if (pCompInfo) free(pCompInfo); + + return 0; +} + +static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey) { + if (pIter == NULL) return 0; + + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) return 0; + + SDataRow row = SL_GET_NODE_DATA(node); + if (dataRowKey(row) >= minKey && dataRowKey(row) <= maxKey) return 1; + + return 0; +} + +static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) { + for (int i = 0; i < nIters; i++) { + SSkipListIterator *pIter = iters[i]; + if (tsdbHasDataInRange(pIter, minKey, maxKey)) return 1; + } + return 0; +} + +static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, int64_t uid) { + size_t size = sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols; + SCompData *pCompData = (SCompData *)malloc(size); + if (pCompData == NULL) return -1; + + pCompData->delimiter = TSDB_FILE_DELIMITER; + pCompData->uid = uid; + pCompData->numOfCols = pCols->numOfCols; + + *offset = lseek(pFile->fd, 0, SEEK_END); + *len = size; + + int toffset = size; + for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { + SCompCol *pCompCol = pCompData->cols + iCol; + SDataCol *pDataCol = pCols->cols + iCol; + + pCompCol->colId = pDataCol->colId; + pCompCol->type = pDataCol->type; + pCompCol->offset = toffset; + + // TODO: add compression + pCompCol->len = TYPE_BYTES[pCompCol->type] * pointsToWrite; + toffset += pCompCol->len; + } + + // Write the block + if (write(pFile->fd, (void *)pCompData, size) < 0) goto _err; + *len += size; + for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { + SDataCol *pDataCol = pCols->cols + iCol; + SCompCol *pCompCol = pCompData->cols + iCol; + if (write(pFile->fd, pDataCol->pData, pCompCol->len) < 0) goto _err; + *len += pCompCol->len; + } + + if (pCompData == NULL) free((void *)pCompData); + return 0; + +_err: + if (pCompData == NULL) free((void *)pCompData); + return -1; +} + +static int compareKeyBlock(const void *arg1, const void *arg2) { + TSKEY key = *(TSKEY *)arg1; + SCompBlock *pBlock = (SCompBlock *)arg2; + + if (key < pBlock->keyFirst) { + return -1; + } else if (key > pBlock->keyLast) { + return 1; + } + + return 0; +} + +int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) { + STsdbCfg * pCfg = &(pRepo->config); + SCompData *pCompData = NULL; + SFile * pFile = NULL; + int numOfPointsToWrite = 0; + int64_t offset = 0; + int32_t len = 0; + + memset((void *)pCompBlock, 0, sizeof(SCompBlock)); + + if (pCompInfo == NULL) { + // Just append the data block to .data or .l or .last file + numOfPointsToWrite = pCols->numOfPoints; + if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // Write to .data file + pFile = &(pGroup->files[TSDB_FILE_TYPE_DATA]); + } else { // Write to .last or .l file + pCompBlock->last = 1; + if (lFile) { + pFile = lFile; + } else { + pFile = &(pGroup->files[TSDB_FILE_TYPE_LAST]); + } + } + tsdbWriteBlockToFileImpl(pFile, pCols, numOfPointsToWrite, &offset, &len, uid); + pCompBlock->offset = offset; + pCompBlock->len = len; + pCompBlock->algorithm = 2; // TODO : add to configuration + pCompBlock->sversion = pCols->sversion; + pCompBlock->numOfPoints = pCols->numOfPoints; + pCompBlock->numOfSubBlocks = 1; + pCompBlock->numOfCols = pCols->numOfCols; + pCompBlock->keyFirst = dataColsKeyFirst(pCols); + pCompBlock->keyLast = dataColsKeyLast(pCols); + } else { + // Need to merge the block to either the last block or the other block + TSKEY keyFirst = dataColsKeyFirst(pCols); + SCompBlock *pMergeBlock = NULL; + + // Search the block to merge in + void *ptr = taosbsearch((void *)&keyFirst, (void *)(pCompInfo->blocks), sizeof(SCompBlock), pIdx->numOfSuperBlocks, + compareKeyBlock, TD_GE); + if (ptr == NULL) { + // No block greater or equal than the key, but there are data in the .last file, need to merge the last file block + // and merge the data + pMergeBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks - 1); + } else { + pMergeBlock = (SCompBlock *)ptr; + } + + if (pMergeBlock->last) { + if (pMergeBlock->last + pCols->numOfPoints > pCfg->minRowsPerFileBlock) { + // Need to load the data from .last and combine data in pCols to write to .data file + + } else { // Just append the block to .last or .l file + if (lFile) { + // read the block from .last file and merge with pCols, write to .l file + + } else { + // tsdbWriteBlockToFileImpl(); + } + } + } else { // The block need to merge in .data file + + } + + } + + return numOfPointsToWrite; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 98dcd45bedece4351a67430a213c597663fb35cd..22680a839b28db0d26723df94eedcb58a1758abe 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -18,7 +18,6 @@ static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable); static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable); static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable); static int tsdbEstimateTableEncodeSize(STable *pTable); -static char * getTupleKey(const void *data); /** * Encode a TSDB table object as a binary content @@ -102,12 +101,9 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { if (pTable == NULL) return -1; if (pTable->type == TSDB_SUPER_TABLE) { - pTable->content.pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, getTupleKey); - } else { - pTable->content.pData = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); - } + pTable->pIndex = + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, 0, getTupleKey); + } tsdbAddTableToMeta(pMeta, pTable, false); @@ -137,6 +133,8 @@ STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables) { pMeta->nTables = 0; pMeta->superList = NULL; pMeta->tables = (STable **)calloc(maxTables, sizeof(STable *)); + pMeta->maxRowBytes = 0; + pMeta->maxCols = 0; if (pMeta->tables == NULL) { free(pMeta); return NULL; @@ -208,10 +206,10 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { super->schema = tdDupSchema(pCfg->schema); super->tagSchema = tdDupSchema(pCfg->tagSchema); super->tagVal = tdDataRowDup(pCfg->tagValues); - super->content.pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, - 0, getTupleKey); // Allow duplicate key, no lock + super->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, + 0, 0, getTupleKey); // Allow duplicate key, no lock - if (super->content.pIndex == NULL) { + if (super->pIndex == NULL) { tdFreeSchema(super->schema); tdFreeSchema(super->tagSchema); tdFreeDataRow(super->tagVal); @@ -223,7 +221,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { } } - STable *table = (STable *)malloc(sizeof(STable)); + STable *table = (STable *)calloc(1, sizeof(STable)); if (table == NULL) { if (newSuper) tsdbFreeTable(super); return -1; @@ -238,8 +236,11 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { table->type = TSDB_NORMAL_TABLE; table->superUid = -1; table->schema = tdDupSchema(pCfg->schema); + if (schemaNCols(table->schema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(table->schema); + tdUpdateSchema(table->schema); + int bytes = tdMaxRowBytesFromSchema(table->schema); + if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes; } - table->content.pData = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); // Register to meta if (newSuper) tsdbAddTableToMeta(pMeta, super, true); @@ -299,10 +300,10 @@ int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) { return 0; } -int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable) { - tSkipListPut(pTable->content.pData, pNode); - return 0; -} +// int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable) { +// tSkipListPut(pTable->mem->pData, pNode); +// return 0; +// } static int tsdbFreeTable(STable *pTable) { // TODO: finish this function @@ -314,10 +315,8 @@ static int tsdbFreeTable(STable *pTable) { // Free content if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) { - tSkipListDestroy(pTable->content.pIndex); - } else { - tSkipListDestroy(pTable->content.pData); - } + tSkipListDestroy(pTable->pIndex); + } free(pTable); return 0; @@ -350,10 +349,10 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) { } else { // add non-super table to the array pMeta->tables[pTable->tableId.tid] = pTable; - if (pTable->type == TSDB_CHILD_TABLE) { - // add STABLE to the index + if (pTable->type == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index tsdbAddTableIntoIndex(pMeta, pTable); } + pMeta->nTables++; } @@ -374,8 +373,27 @@ static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable) { return 0; } static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { - assert(pTable->type == TSDB_CHILD_TABLE); - // TODO + assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); + STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); + assert(pSTable != NULL); + + int32_t level = 0; + int32_t headSize = 0; + + // first tag column + STColumn* s = schemaColAt(pSTable->tagSchema, 0); + + tSkipListRandNodeInfo(pSTable->pIndex, &level, &headSize); + SSkipListNode* pNode = calloc(1, headSize + s->bytes + POINTER_BYTES); + pNode->level = level; + + SSkipList* list = pSTable->pIndex; + + memcpy(SL_GET_NODE_KEY(list, pNode), dataRowTuple(pTable->tagVal), colBytes(s)); + memcpy(SL_GET_NODE_DATA(pNode), &pTable, POINTER_BYTES); + + tSkipListPut(list, pNode); + return 0; } @@ -404,7 +422,7 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) { return size; } -static char *getTupleKey(const void * data) { +char *getTupleKey(const void * data) { SDataRow row = (SDataRow)data; return dataRowAt(row, TD_DATA_ROW_HEAD_SIZE); diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index a62299c45fa83b7e021e2e91863e10eea9126046..2919b2cf9e9229ecc7bf558cb484833d5dc2cb28 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -14,53 +14,1123 @@ */ #include "os.h" + +#include "tlog.h" +#include "tutil.h" + +#include "../../../query/inc/qast.h" +#include "../../../query/inc/tsqlfunction.h" #include "tsdb.h" +#include "tsdbFile.h" +#include "tsdbMeta.h" + +#define EXTRA_BYTES 2 +#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoEx *)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) +#define QUERY_IS_ASC_QUERY(o) (o == TSQL_SO_ASC) +#define QH_GET_NUM_OF_COLS(handle) (taosArrayGetSize((handle)->pColumns)) + +enum { + QUERY_RANGE_LESS_EQUAL = 0, + QUERY_RANGE_GREATER_EQUAL = 1, +}; + +typedef struct SField { + // todo need the definition +} SField; + +typedef struct SHeaderFileInfo { + int32_t fileId; +} SHeaderFileInfo; + +typedef struct SQueryFilePos { + int32_t fid; + int32_t slot; + int32_t pos; + int64_t lastKey; +} SQueryFilePos; + +typedef struct SDataBlockLoadInfo { + int32_t fileListIndex; + int32_t fileId; + int32_t slotIdx; + int32_t sid; + SArray *pLoadedCols; +} SDataBlockLoadInfo; + +typedef struct SLoadCompBlockInfo { + int32_t sid; /* meter sid */ + int32_t fileId; + int32_t fileListIndex; +} SLoadCompBlockInfo; + +typedef struct SQueryFilesInfo { + SArray *pFileInfo; + int32_t current; // the memory mapped header file, NOTE: only one header file can be mmap. + int32_t vnodeId; + + int32_t headerFd; // header file fd + int64_t headerFileSize; + int32_t dataFd; + int32_t lastFd; + + char headerFilePath[PATH_MAX]; // current opened header file name + char dataFilePath[PATH_MAX]; // current opened data file name + char lastFilePath[PATH_MAX]; // current opened last file path + char dbFilePathPrefix[PATH_MAX]; +} SQueryFilesInfo; + +typedef struct STableCheckInfo { + STableId tableId; + TSKEY lastKey; + STable * pTableObj; + int64_t offsetInHeaderFile; +// int32_t numOfBlocks; + int32_t start; + bool checkFirstFileBlock; + + SCompIdx* compIndex; + + SCompBlock *pBlock; + SSkipListIterator* iter; +} STableCheckInfo; + +typedef struct { + SCompBlock *compBlock; + SField * fields; +} SCompBlockFields; + +typedef struct STableDataBlockInfoEx { + SCompBlockFields pBlock; + STableCheckInfo* pMeterDataInfo; + int32_t blockIndex; + int32_t groupIdx; /* number of group is less than the total number of meters */ +} STableDataBlockInfoEx; + +enum { + SINGLE_TABLE_MODEL = 1, + MULTI_TABLE_MODEL = 2, +}; + +typedef struct STsdbQueryHandle { + struct STsdbRepo* pTsdb; + int8_t model; // access model, single table model or multi-table model + SQueryFilePos cur; // current position + SQueryFilePos start; // the start position, used for secondary/third iteration + int32_t unzipBufSize; + char *unzipBuffer; + char *secondaryUnzipBuffer; + + SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ + SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ + SQueryFilesInfo vnodeFileInfo; + + int16_t numOfRowsPerPage; + uint16_t flag; // denotes reversed scan of data or not + int16_t order; + STimeWindow window; // the primary query time window that applies to all queries + int32_t blockBufferSize; + SCompBlock* pBlock; + int32_t numOfBlocks; + SField ** pFields; + SArray * pColumns; // column list, SColumnInfoEx array list + bool locateStart; + int32_t realNumOfRows; + bool loadDataAfterSeek; // load data after seek. + SArray* pTableCheckInfo; + int32_t activeIndex; + + int32_t tableIndex; + bool isFirstSlot; + void * qinfo; // query info handle, for debug purpose + + STableDataBlockInfoEx *pDataBlockInfoEx; +} STsdbQueryHandle; + +int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) { + // record the maximum column width among columns of this meter/metric + SColumnInfoEx *pColumn = taosArrayGet(pQueryHandle->pColumns, 0); + + int32_t maxColWidth = pColumn->info.bytes; + for (int32_t i = 1; i < QH_GET_NUM_OF_COLS(pQueryHandle); ++i) { + int32_t bytes = pColumn[i].info.bytes; + if (bytes > maxColWidth) { + maxColWidth = bytes; + } + } + + // only one unzip buffer required, since we can unzip each column one by one + pQueryHandle->unzipBufSize = (size_t)(maxColWidth * rowsPerFileBlock + EXTRA_BYTES); // plus extra_bytes + pQueryHandle->unzipBuffer = (char *)calloc(1, pQueryHandle->unzipBufSize); + + pQueryHandle->secondaryUnzipBuffer = (char *)calloc(1, pQueryHandle->unzipBufSize); + + if (pQueryHandle->unzipBuffer == NULL || pQueryHandle->secondaryUnzipBuffer == NULL) { + goto _error_clean; + } + + return TSDB_CODE_SUCCESS; -tsdb_query_handle_t *tsdbQueryByTableId(STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo) { +_error_clean: + tfree(pQueryHandle->unzipBuffer); + tfree(pQueryHandle->secondaryUnzipBuffer); + return TSDB_CODE_SERV_OUT_OF_MEMORY; } -bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { +static void initQueryFileInfoFD(SQueryFilesInfo *pVnodeFilesInfo) { + pVnodeFilesInfo->current = -1; + pVnodeFilesInfo->headerFileSize = -1; + + pVnodeFilesInfo->headerFd = FD_INITIALIZER; // set the initial value + pVnodeFilesInfo->dataFd = FD_INITIALIZER; + pVnodeFilesInfo->lastFd = FD_INITIALIZER; +} + +static void vnodeInitDataBlockLoadInfo(SDataBlockLoadInfo *pBlockLoadInfo) { + pBlockLoadInfo->slotIdx = -1; + pBlockLoadInfo->fileId = -1; + pBlockLoadInfo->sid = -1; + pBlockLoadInfo->fileListIndex = -1; +} + +static void vnodeInitCompBlockLoadInfo(SLoadCompBlockInfo *pCompBlockLoadInfo) { + pCompBlockLoadInfo->sid = -1; + pCompBlockLoadInfo->fileId = -1; + pCompBlockLoadInfo->fileListIndex = -1; +} + +static int fileOrderComparFn(const void *p1, const void *p2) { + SHeaderFileInfo *pInfo1 = (SHeaderFileInfo *)p1; + SHeaderFileInfo *pInfo2 = (SHeaderFileInfo *)p2; + + if (pInfo1->fileId == pInfo2->fileId) { + return 0; + } + + return (pInfo1->fileId > pInfo2->fileId) ? 1 : -1; +} + +void vnodeRecordAllFiles(int32_t vnodeId, SQueryFilesInfo *pVnodeFilesInfo) { + char suffix[] = ".head"; + pVnodeFilesInfo->pFileInfo = taosArrayInit(4, sizeof(int32_t)); + + struct dirent *pEntry = NULL; + pVnodeFilesInfo->vnodeId = vnodeId; + char* tsDirectory = ""; + + sprintf(pVnodeFilesInfo->dbFilePathPrefix, "%s/vnode%d/db/", tsDirectory, vnodeId); + DIR *pDir = opendir(pVnodeFilesInfo->dbFilePathPrefix); + if (pDir == NULL) { + // dError("QInfo:%p failed to open directory:%s, %s", pQInfo, pVnodeFilesInfo->dbFilePathPrefix, + // strerror(errno)); + return; + } + + while ((pEntry = readdir(pDir)) != NULL) { + if ((pEntry->d_name[0] == '.' && pEntry->d_name[1] == '\0') || (strcmp(pEntry->d_name, "..") == 0)) { + continue; + } + + if (pEntry->d_type & DT_DIR) { + continue; + } + + size_t len = strlen(pEntry->d_name); + if (strcasecmp(&pEntry->d_name[len - 5], suffix) != 0) { + continue; + } + + int32_t vid = 0; + int32_t fid = 0; + sscanf(pEntry->d_name, "v%df%d", &vid, &fid); + if (vid != vnodeId) { /* ignore error files */ + // dError("QInfo:%p error data file:%s in vid:%d, ignore", pQInfo, pEntry->d_name, vnodeId); + continue; + } + +// int32_t firstFid = pVnode->fileId - pVnode->numOfFiles + 1; +// if (fid > pVnode->fileId || fid < firstFid) { +// dError("QInfo:%p error data file:%s in vid:%d, fid:%d, fid range:%d-%d", pQInfo, pEntry->d_name, vnodeId, +// fid, firstFid, pVnode->fileId); +// continue; +// } + + assert(fid >= 0 && vid >= 0); + taosArrayPush(pVnodeFilesInfo->pFileInfo, &fid); + } + + closedir(pDir); + + // dTrace("QInfo:%p find %d data files in %s to be checked", pQInfo, pVnodeFilesInfo->numOfFiles, + // pVnodeFilesInfo->dbFilePathPrefix); + + // order the files information according their names */ + size_t numOfFiles = taosArrayGetSize(pVnodeFilesInfo->pFileInfo); + qsort(pVnodeFilesInfo->pFileInfo->pData, numOfFiles, sizeof(SHeaderFileInfo), fileOrderComparFn); +} + +tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo) { + // todo 1. filter not exist table + + // todo 2. add the reference count for each table that is involved in query + + STsdbQueryHandle *pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); + pQueryHandle->order = pCond->order; + pQueryHandle->window = pCond->twindow; + pQueryHandle->pTsdb = tsdb; + + pQueryHandle->pColumns = pColumnInfo; + pQueryHandle->loadDataAfterSeek = false; + pQueryHandle->isFirstSlot = true; + + size_t size = taosArrayGetSize(idList); + assert(size >= 1); + + pQueryHandle->pTableCheckInfo = taosArrayInit(size, sizeof(STableCheckInfo)); + for(int32_t i = 0; i < size; ++i) { + STableId id = *(STableId*) taosArrayGet(idList, i); + + STableCheckInfo info = { + .lastKey = pQueryHandle->window.skey, + .tableId = id, + .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), //todo this may be failed + }; + + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + } + + pQueryHandle->model = (size > 1)? MULTI_TABLE_MODEL:SINGLE_TABLE_MODEL; + + pQueryHandle->activeIndex = 0; + + // malloc buffer in order to load data from file + int32_t numOfCols = taosArrayGetSize(pColumnInfo); + size_t bufferCapacity = 4096; + + pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoEx)); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoEx *pCol = taosArrayGet(pColumnInfo, i); + SColumnInfoEx pDest = {{0}, 0}; + + pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCol->info.bytes); + pDest.info = pCol->info; + taosArrayPush(pQueryHandle->pColumns, &pDest); + } + + if (doAllocateBuf(pQueryHandle, bufferCapacity) != TSDB_CODE_SUCCESS) { + return NULL; + } + + initQueryFileInfoFD(&pQueryHandle->vnodeFileInfo); + vnodeInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); + vnodeInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + + int32_t vnodeId = 1; + vnodeRecordAllFiles(vnodeId, &pQueryHandle->vnodeFileInfo); + + return (tsdb_query_handle_t)pQueryHandle; +} + +static bool hasMoreDataInCacheForSingleModel(STsdbQueryHandle* pHandle) { + assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); + + STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + STable *pTable = pTableCheckInfo->pTableObj; + assert(pTable != NULL); + + // no data in cache, abort + if (pTable->mem == NULL && pTable->imem == NULL) { + return false; + } + + // all data in mem are checked already. + if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { + return false; + } + + return true; +} + +// todo dynamic get the daysperfile +static int32_t getFileIdFromKey(TSKEY key) { + return (int32_t)(key / 10); // set the starting fileId +} + +static int32_t getFileCompInfo(STableCheckInfo* pCheckInfo, SFileGroup* fileGroup) { + tsdbLoadCompIdx(fileGroup, pCheckInfo->compIndex, 10000); // todo set dynamic max tables + SCompIdx* compIndex = &pCheckInfo->compIndex[pCheckInfo->tableId.tid]; + + if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file + + } else { + tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pBlock); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t binarySearchForBlockImpl(SCompBlock *pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { + int32_t firstSlot = 0; + int32_t lastSlot = numOfBlocks - 1; + + int32_t midSlot = firstSlot; + + while (1) { + numOfBlocks = lastSlot - firstSlot + 1; + midSlot = (firstSlot + (numOfBlocks >> 1)); + + if (numOfBlocks == 1) break; + + if (skey > pBlock[midSlot].keyLast) { + if (numOfBlocks == 2) break; + if ((order == TSQL_SO_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; + firstSlot = midSlot + 1; + } else if (skey < pBlock[midSlot].keyFirst) { + if ((order == TSQL_SO_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; + lastSlot = midSlot - 1; + } else { + break; // got the slot + } + } + + return midSlot; +} + +static SDataBlockInfo getTrueBlockInfo(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { + SDataBlockInfo info = {{0}, 0}; + + SCompBlock *pDiskBlock = &pCheckInfo->pBlock[pHandle->cur.slot]; + + info.window.skey = pDiskBlock->keyFirst; + info.window.ekey = pDiskBlock->keyLast; + info.size = pDiskBlock->numOfPoints; + info.numOfCols = pDiskBlock->numOfCols; + + return info; +} + +bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { + SQueryFilePos *cur = &pQueryHandle->cur; + + if (pQueryHandle->cur.fid >= 0) { + int32_t fileIndex = -1; + + /* + * 1. ascending order. The last data block of data file + * 2. descending order. The first block of file + */ + if ((step == QUERY_ASC_FORWARD_STEP && (pQueryHandle->cur.slot == pQueryHandle->numOfBlocks - 1)) || + (step == QUERY_DESC_FORWARD_STEP && (pQueryHandle->cur.slot == 0))) { + // temporarily keep the position value, in case of no data qualified when move forwards(backwards) + SQueryFilePos save = pQueryHandle->cur; + +// fileIndex = getNextDataFileCompInfo(pQueryHandle, &pQueryHandle->cur, &pQueryHandle->vnodeFileInfo, step); + + // first data block in the next file + if (fileIndex >= 0) { + cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; + cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; +// return loadQaulifiedData(pQueryHandle); + } else {// try data in cache + assert(cur->fid == -1); + + if (step == QUERY_ASC_FORWARD_STEP) { +// TSKEY nextTimestamp = +// getQueryStartPositionInCache_rv(pQueryHandle, &pQueryHandle->cur.slot, &pQueryHandle->cur.pos, true); +// if (nextTimestamp < 0) { +// pQueryHandle->cur = save; +// } +// +// return (nextTimestamp > 0); + } + + // no data to check for desc order query, restore the saved position value + pQueryHandle->cur = save; + return false; + } + } + + // next block in the same file + int32_t fid = cur->fid; +// fileIndex = vnodeGetVnodeHeaderFileIndex(&fid, pQueryHandle->order, &pQueryHandle->vnodeFileInfo); + cur->slot += step; + + SCompBlock *pBlock = &pQueryHandle->pBlock[cur->slot]; + cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; +// return loadQaulifiedData(pQueryHandle); + } else { // data in cache + return hasMoreDataInCacheForSingleModel(pQueryHandle); + } +} + +int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { + int firstPos, lastPos, midPos = -1; + int numOfPoints; + TSKEY *keyList; + + if (num <= 0) return -1; + + keyList = (TSKEY *)pValue; + firstPos = 0; + lastPos = num - 1; + + if (order == 0) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[lastPos]) return lastPos; + if (key == keyList[firstPos]) return firstPos; + if (key < keyList[firstPos]) return firstPos - 1; + + numOfPoints = lastPos - firstPos + 1; + midPos = (numOfPoints >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfPoints = lastPos - firstPos + 1; + midPos = (numOfPoints >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + +static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SArray *sa) { + // only return the qualified data to client in terms of query time window, data rows in the same block but do not + // be included in the query time window will be discarded + SQueryFilePos *cur = &pQueryHandle->cur; + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + SDataBlockInfo blockInfo = getTrueBlockInfo(pQueryHandle, pCheckInfo); + + int32_t endPos = cur->pos; + if (QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { + endPos = blockInfo.size - 1; + pQueryHandle->realNumOfRows = endPos - cur->pos + 1; + } else if (!QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { + endPos = 0; + pQueryHandle->realNumOfRows = cur->pos + 1; + } else { +// endPos = vnodeBinarySearchKey(pQueryHandle->tsBuf->data, blockInfo.size, pQueryHandle->window.ekey, pQueryHandle->order); + + if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (endPos < cur->pos) { + pQueryHandle->realNumOfRows = 0; + return; + } else { + pQueryHandle->realNumOfRows = endPos - cur->pos; + } + } else { + if (endPos > cur->pos) { + pQueryHandle->realNumOfRows = 0; + return; + } else { + pQueryHandle->realNumOfRows = cur->pos - endPos; + } + } + } + + int32_t start = MIN(cur->pos, endPos); + + // move the data block in the front to data block if needed + if (start != 0) { + int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + + for (int32_t i = 0; i < taosArrayGetSize(sa); ++i) { + int16_t colId = *(int16_t *)taosArrayGet(sa, i); + + for (int32_t j = 0; j < numOfCols; ++j) { + SColumnInfoEx *pCol = taosArrayGet(pQueryHandle->pColumns, j); + + if (pCol->info.colId == colId) { + memmove(pCol->pData, ((char *)pCol->pData) + pCol->info.bytes * start, pQueryHandle->realNumOfRows * pCol->info.bytes); + break; + } + } + } + } + + assert(pQueryHandle->realNumOfRows <= blockInfo.size); + + // forward(backward) the position for cursor + cur->pos = endPos; +} + +static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { + STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); + int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); + + SFileGroup* fileGroup = tsdbSearchFGroup(pFileHandle, fid); + if (fileGroup == NULL) { + return false; + } + + SQueryFilePos* cur = &pQueryHandle->cur; + + TSKEY key = pCheckInfo->lastKey; + int32_t index = -1; + + // todo add iterator for filegroup + while (1) { + if ((fid = getFileCompInfo(pCheckInfo, fileGroup)) < 0) { + break; + } + + int32_t tid = pCheckInfo->tableId.tid; + index = binarySearchForBlockImpl(pCheckInfo->pBlock, pCheckInfo->compIndex[tid].numOfSuperBlocks, pQueryHandle->order, key); + + if (type == QUERY_RANGE_GREATER_EQUAL) { + if (key <= pCheckInfo->pBlock[index].keyLast) { + break; + } else { + index = -1; + } + } else { + if (key >= pCheckInfo->pBlock[index].keyFirst) { + break; + } else { + index = -1; + } + } + } + + // failed to find qualified point in file, abort + if (index == -1) { + return false; + } + + assert(index >= 0 && index < pQueryHandle->numOfBlocks); + + // load first data block into memory failed, caused by disk block error + bool blockLoaded = false; + SArray *sa = NULL; + + // todo no need to loaded at all + cur->slot = index; + +// sa = getDefaultLoadColumns(pQueryHandle, true); + if (tsdbLoadDataBlock(&fileGroup->files[2], &pCheckInfo->pBlock[cur->slot], 1, fid, sa) == 0) { + blockLoaded = true; + } + + // dError("QInfo:%p fileId:%d total numOfBlks:%d blockId:%d load into memory failed due to error in disk files", + // GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx); + + // failed to load data from disk, abort current query + if (blockLoaded == false) { + return false; + } + + // todo search qualified points in blk, according to primary key (timestamp) column +// cur->pos = binarySearchForBlockImpl(ptsBuf->data, pBlocks->numOfPoints, key, pQueryHandle->order); + assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); + + filterDataInDataBlock(pQueryHandle, sa); + return pQueryHandle->realNumOfRows > 0; +} + +static bool hasMoreDataInFileForSingleTableModel(STsdbQueryHandle* pHandle) { + assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); + + STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); + SQueryFilePos* cur = &pHandle->cur; + + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + if (!pCheckInfo->checkFirstFileBlock) { + pCheckInfo->checkFirstFileBlock = true; + + if (pFileHandle != NULL) { + bool found = getQualifiedDataBlock(pHandle, pCheckInfo, 1); + if (found) { + return true; + } + } + + // no data in file, try cache + pHandle->cur.fid = -1; + return hasMoreDataInCacheForSingleModel(pHandle); + } else { // move to next data block in file or in cache + return moveToNextBlock(pHandle, 1); + } +} + +static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) { + size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo); + assert(numOfTables > 0); + + while(pHandle->activeIndex < numOfTables) { + STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + STable *pTable = pTableCheckInfo->pTableObj; + if (pTable->mem == NULL && pTable->imem == NULL) { + pHandle->activeIndex += 1; // try next table if exits + continue; + } + + // all data in mem are checked already. + if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { + pHandle->activeIndex += 1; // try next table if exits + continue; + } + + return true; + } + + // all tables has checked already return false; } -SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { +// handle data in cache situation +bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; + if (pHandle->model == SINGLE_TABLE_MODEL) { + return hasMoreDataInFileForSingleTableModel(pHandle); + } else { + return hasMoreDataInCacheForMultiModel(pHandle); + } +} +static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, + TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pHandle) { + int numOfRows = 0; + int32_t numOfCols = taosArrayGetSize(pHandle->pColumns); + *skey = INT64_MIN; + + while(tSkipListIterNext(pIter)) { + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) break; + + SDataRow row = SL_GET_NODE_DATA(node); + if (dataRowKey(row) > maxKey) break; + + if (*skey == INT64_MIN) { + *skey = dataRowKey(row); + } + + *ekey = dataRowKey(row); + + int32_t offset = 0; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoEx* pColInfo = taosArrayGet(pHandle->pColumns, i); + memcpy(pColInfo->pData + numOfRows*pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); + offset += pColInfo->info.bytes; + } + + numOfRows++; + if (numOfRows >= maxRowsToRead) break; + }; + + return numOfRows; } -int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis) { +// copy data from cache into data block +SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; + + STableCheckInfo* pTableQInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + STable *pTable = pTableQInfo->pTableObj; + + TSKEY skey = 0, ekey = 0; + int32_t rows = 0; + + if (pTable->mem != NULL) { + + // create mem table iterator if it is not created yet + if (pTableQInfo->iter == NULL) { + pTableQInfo->iter = tSkipListCreateIter(pTable->mem->pData); + } + + rows = tsdbReadRowsFromCache(pTableQInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle); + } + + SDataBlockInfo blockInfo = { + .uid = pTable->tableId.uid, + .sid = pTable->tableId.tid, + .size = rows, + .window = {.skey = skey, .ekey = ekey} + }; + + // update the last key value + pTableQInfo->lastKey = ekey + 1; + + return blockInfo; +} +// return null for data block in cache +int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis) { + *pBlockStatis = NULL; + return TSDB_CODE_SUCCESS; } SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList) { + // in case of data in cache, all data has been kept in column info object. + STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; + return pHandle->pColumns; +} + +int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow *window, tsdbpos_t position, int16_t order) {} + +int32_t tsdbDataBlockSeek(tsdb_query_handle_t *pQueryHandle, tsdbpos_t pos) {} + +tsdbpos_t tsdbDataBlockTell(tsdb_query_handle_t *pQueryHandle) { return NULL; } + +SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond) {} + +tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr) {} + +SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) {} + +static SArray* createTableIdArrayList(struct STsdbRepo* tsdb, int64_t uid) { + STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); + assert(pTable != NULL); //assert pTable is a super table + + size_t size = tSkipListGetSize(pTable->pIndex); + SArray* pList = taosArrayInit(size, sizeof(STableId)); + + SSkipListIterator* iter = tSkipListCreateIter(pTable->pIndex); + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + STable* t = *(STable**) SL_GET_NODE_DATA(pNode); + + taosArrayPush(pList, &t->tableId); + } + + return pList; +} + +typedef struct SSyntaxTreeFilterSupporter { + SSchema* pTagSchema; + int32_t numOfTags; + int32_t optr; +} SSyntaxTreeFilterSupporter; +/** + * convert the result pointer to STabObj instead of tSkipListNode + * @param pRes + */ +static void tansformQueryResult(SArray* pRes) { + if (pRes == NULL || taosArrayGetSize(pRes) == 0) { + return; + } + + size_t size = taosArrayGetSize(pRes); + for (int32_t i = 0; i < size; ++i) { +// pRes->pRes[i] = ((tSkipListNode*)(pRes->pRes[i]))->pData; + } +} + +void tSQLListTraverseDestroyInfo(void* param) { + if (param == NULL) { + return; + } + + tQueryInfo* pInfo = (tQueryInfo*)param; + tVariantDestroy(&(pInfo->q)); + free(param); } -int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow* window, tsdbpos_t position, int16_t order) { +static char* convertTagQueryStr(const wchar_t* str, size_t len) { + char* mbs = NULL; + + if (len > 0) { + mbs = calloc(1, (len + 1) * TSDB_NCHAR_SIZE); + taosUcs4ToMbs((void*) str, len * TSDB_NCHAR_SIZE, mbs); //todo add log + } + + return mbs; +} + +static int32_t compareStrVal(const void* pLeft, const void* pRight) { + int32_t ret = strcmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} +static int32_t compareWStrVal(const void* pLeft, const void* pRight) { + int32_t ret = wcscmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } } -int32_t tsdbDataBlockSeek(tsdb_query_handle_t *pQueryHandle, tsdbpos_t pos) { +static int32_t compareIntVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_INT64_VAL(pRight)); +} +static int32_t compareIntDoubleVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_DOUBLE_VAL(pRight)); } -tsdbpos_t tsdbDataBlockTell(tsdb_query_handle_t *pQueryHandle) { - return NULL; +static int32_t compareDoubleVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_DOUBLE_VAL(pLeft), GET_DOUBLE_VAL(pRight)); } -SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond) { +static int32_t compareDoubleIntVal(const void* pLeft, const void* pRight) { + double ret = (*(double*)pLeft) - (*(int64_t*)pRight); + if (fabs(ret) < DBL_EPSILON) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} +static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const char* pattern = pRight; + const char* str = pLeft; + + int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr) { +static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const wchar_t* pattern = pRight; + const wchar_t* str = pLeft; + + int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} +static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { + __compar_fn_t comparator = NULL; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_BOOL: { + if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { + comparator = compareIntVal; + } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { + comparator = compareIntDoubleVal; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { + comparator = compareDoubleIntVal; + } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { + comparator = compareDoubleVal; + } + break; + } + + case TSDB_DATA_TYPE_BINARY: { + assert(filterType == TSDB_DATA_TYPE_BINARY); + + if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + comparator = compareStrPatternComp; + } else { /* normal relational comparator */ + comparator = compareStrVal; + } + + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + assert(filterType == TSDB_DATA_TYPE_NCHAR); + + if (optr == TSDB_RELATION_LIKE) { + comparator = compareWStrPatternComp; + } else { + comparator = compareWStrVal; + } + + break; + } + default: + comparator = compareIntVal; + break; + } + + return comparator; } -STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) { +static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pSchema, int32_t* index, + int32_t* offset) { + *index = 0; + *offset = 0; + + // filter on table name(TBNAME) + if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { + *index = TSDB_TBNAME_COLUMN_INDEX; + *offset = TSDB_TBNAME_COLUMN_INDEX; + return; + } + + while ((*index) < pSupporter->numOfTags) { + if (pSupporter->pTagSchema[*index].bytes == pSchema->bytes && + pSupporter->pTagSchema[*index].type == pSchema->type && + strcmp(pSupporter->pTagSchema[*index].name, pSchema->name) == 0) { + break; + } else { + (*offset) += pSupporter->pTagSchema[(*index)++].bytes; + } + } +} +void filterPrepare(void* expr, void* param) { + tSQLBinaryExpr *pExpr = (tSQLBinaryExpr*) expr; + if (pExpr->info != NULL) { + return; + } + + int32_t i = 0, offset = 0; + pExpr->info = calloc(1, sizeof(tQueryInfo)); + + tQueryInfo* pInfo = pExpr->info; + SSyntaxTreeFilterSupporter* pSupporter = (SSyntaxTreeFilterSupporter*)param; + + tVariant* pCond = pExpr->pRight->pVal; + SSchema* pSchema = pExpr->pLeft->pSchema; + + getTagColumnInfo(pSupporter, pSchema, &i, &offset); + assert((i >= 0 && i < TSDB_MAX_TAGS) || (i == TSDB_TBNAME_COLUMN_INDEX)); + assert((offset >= 0 && offset < TSDB_MAX_TAGS_LEN) || (offset == TSDB_TBNAME_COLUMN_INDEX)); + + pInfo->sch = *pSchema; + pInfo->colIdx = i; + pInfo->optr = pExpr->nSQLBinaryOptr; + pInfo->offset = offset; + pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr); + + tVariantAssign(&pInfo->q, pCond); + tVariantTypeSetType(&pInfo->q, pInfo->sch.type); } -STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond) { +bool tSkipListNodeFilterCallback(const void* pNode, void* param) { + tQueryInfo* pInfo = (tQueryInfo*)param; + + STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); + + char* val = dataRowTuple(pTable->tagVal); // todo not only the first column + int8_t type = pInfo->sch.type; + + int32_t ret = 0; + if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) { + ret = pInfo->compare(val, pInfo->q.pz); + } else { + tVariant t = {0}; + tVariantCreateFromBinary(&t, val, (uint32_t) pInfo->sch.bytes, type); + + ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); + } + + switch (pInfo->optr) { + case TSDB_RELATION_EQUAL: { + return ret == 0; + } + case TSDB_RELATION_NOT_EQUAL: { + return ret != 0; + } + case TSDB_RELATION_LARGE_EQUAL: { + return ret >= 0; + } + case TSDB_RELATION_LARGE: { + return ret > 0; + } + case TSDB_RELATION_LESS_EQUAL: { + return ret <= 0; + } + case TSDB_RELATION_LESS: { + return ret < 0; + } + case TSDB_RELATION_LIKE: { + return ret == 0; + } + + default: + assert(false); + } + return true; +} +static int32_t doQueryTableList(STable* pSTable, SArray* pRes, const char* pCond) { + STColumn* stcol = schemaColAt(pSTable->tagSchema, 0); + + tSQLBinaryExpr* pExpr = NULL; + tSQLBinaryExprFromString(&pExpr, stcol, schemaNCols(pSTable->tagSchema), pCond, strlen(pCond)); + + // failed to build expression, no result, return immediately + if (pExpr == NULL) { + mError("table:%" PRIu64 ", no result returned, error in super table query expression:%s", pSTable->tableId.uid, pCond); + tfree(pCond); + + return TSDB_CODE_OPS_NOT_SUPPORT; + } + + // query according to the binary expression + SSyntaxTreeFilterSupporter s = {.pTagSchema = stcol, .numOfTags = schemaNCols(pSTable->tagSchema)}; + + SBinaryFilterSupp supp = { + .fp = (__result_filter_fn_t)tSkipListNodeFilterCallback, + .setupInfoFn = (__do_filter_suppl_fn_t)filterPrepare, + .pExtInfo = &s + }; + + tSQLBinaryExprTraverse(pExpr, pSTable->pIndex, pRes, &supp); + tSQLBinaryExprDestroy(&pExpr, tSQLListTraverseDestroyInfo); + + tansformQueryResult(pRes); + + return TSDB_CODE_SUCCESS; } +SArray *tsdbQueryTableList(struct STsdbRepo* tsdb, int64_t uid, const wchar_t *pTagCond, size_t len) { + // no condition, all tables created according to the stable will involved in querying + if (pTagCond == NULL || wcslen(pTagCond) == 0) { + return createTableIdArrayList(tsdb, uid); + } else { + char* str = convertTagQueryStr(pTagCond, len); + SArray* result = taosArrayInit(8, POINTER_BYTES); + + STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); + assert(pSTable != NULL); + + if (doQueryTableList(pSTable, result, str) == TSDB_CODE_SUCCESS) { + return result; + } + } +} diff --git a/src/vnode/tsdb/tests/CMakeLists.txt b/src/vnode/tsdb/tests/CMakeLists.txt index 51c15bce203eb920d7f377d261322047ccd008dc..ee1aaba8cd8c623b29f2d1200c640da2a62dfe6e 100644 --- a/src/vnode/tsdb/tests/CMakeLists.txt +++ b/src/vnode/tsdb/tests/CMakeLists.txt @@ -3,9 +3,4 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) add_executable(tsdbTests ${SOURCE_LIST}) target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb) -add_test( - NAME - unit - COMMAND - ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests -) \ No newline at end of file +add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests) \ No newline at end of file diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 42a22553c7511160c189ab1b5415e29f45dd383e..6cfe0e626dd217b70597c2f75f2f7a575472d1e3 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -1,12 +1,19 @@ #include #include +#include #include "tsdb.h" #include "dataformat.h" #include "tsdbFile.h" #include "tsdbMeta.h" -TEST(TsdbTest, tableEncodeDecode) { +double getCurTime() { + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec + tv.tv_usec * 1E-6; +} + +TEST(TsdbTest, DISABLED_tableEncodeDecode) { STable *pTable = (STable *)malloc(sizeof(STable)); pTable->type = TSDB_NORMAL_TABLE; @@ -40,9 +47,9 @@ TEST(TsdbTest, tableEncodeDecode) { ASSERT_EQ(pTable->superUid, tTable->superUid); ASSERT_EQ(pTable->sversion, tTable->sversion); ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0); - ASSERT_EQ(tTable->content.pData, nullptr); } +// TEST(TsdbTest, DISABLED_createRepo) { TEST(TsdbTest, createRepo) { STsdbCfg config; @@ -72,19 +79,24 @@ TEST(TsdbTest, createRepo) { tsdbCreateTable(pRepo, &tCfg); // // 3. Loop to write some simple data - int nRows = 100; + int nRows = 10000000; int rowsPerSubmit = 10; int64_t start_time = 1584081000000; SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit); + double stime = getCurTime(); + for (int k = 0; k < nRows/rowsPerSubmit; k++) { + memset((void *)pMsg, 0, sizeof(SSubmitMsg)); SSubmitBlk *pBlock = pMsg->blocks; - pBlock->tableId = {.uid = 987607499877672L, .tid = 0}; + pBlock->uid = 987607499877672L; + pBlock->tid = 0; pBlock->sversion = 0; pBlock->len = 0; for (int i = 0; i < rowsPerSubmit; i++) { - start_time += 1000; + // start_time += 1000; + start_time -= 1000; SDataRow row = (SDataRow)(pBlock->data + pBlock->len); tdInitDataRow(row, schema); @@ -99,23 +111,44 @@ TEST(TsdbTest, createRepo) { pBlock->len += dataRowLen(row); } pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; + pMsg->numOfBlocks = 1; + + pBlock->len = htonl(pBlock->len); + pBlock->numOfRows = htonl(pBlock->numOfRows); + pBlock->uid = htobe64(pBlock->uid); + pBlock->tid = htonl(pBlock->tid); + + pBlock->sversion = htonl(pBlock->sversion); + pBlock->padding = htonl(pBlock->padding); + + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + pMsg->compressed = htonl(pMsg->numOfBlocks); tsdbInsertData(pRepo, pMsg); } - tsdbTriggerCommit(pRepo); + double etime = getCurTime(); + + void *ptr = malloc(150000); + free(ptr); + + printf("Spent %f seconds to write %d records\n", etime - stime, nRows); + + tsdbCloseRepo(pRepo); } +// TEST(TsdbTest, DISABLED_openRepo) { TEST(TsdbTest, openRepo) { tsdb_repo_t *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0"); ASSERT_NE(pRepo, nullptr); } -TEST(TsdbTest, createFileGroup) { +TEST(TsdbTest, DISABLED_createFileGroup) { SFileGroup fGroup; - ASSERT_EQ(tsdbCreateFileGroup("/home/ubuntu/work/ttest/vnode0/data", 1820, &fGroup, 1000), 0); + // ASSERT_EQ(tsdbCreateFileGroup("/home/ubuntu/work/ttest/vnode0/data", 1820, &fGroup, 1000), 0); int k = 0; } \ No newline at end of file diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index dc16185d9a1b74cad4492c09f48dc3437772e16e..b621781a3c81e4155841ac5d4ec31aae6a38518e 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -46,7 +46,8 @@ int main(int argc, char *argv[]) { } printf("success to connect to server\n"); - int32_t code = taos_query(taos, "select * from test.t1"); +// int32_t code = taos_query(taos, "insert into test.tm2 values(now, 1)(now+1m,2)(now+2m,3) (now+3m, 4) (now+4m, 5);"); + int32_t code = taos_query(taos, "insert into test.tm2 values(now, 99)"); if (code != 0) { printf("failed to execute query, reason:%s\n", taos_errstr(taos)); } @@ -64,6 +65,9 @@ int main(int argc, char *argv[]) { memset(buf, 0, 512); } + taos_close(taos); + + getchar(); return 0; taos_query(taos, "drop database demo");