diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a05cc0224a1a4c9d82579721cd7870ec81766686..326f00dbbd5c4565c03b3c8b253d511d42b7668c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,7 +12,6 @@ ADD_SUBDIRECTORY(client) ADD_SUBDIRECTORY(query) ADD_SUBDIRECTORY(kit) ADD_SUBDIRECTORY(plugins) -ADD_SUBDIRECTORY(sdb) ADD_SUBDIRECTORY(mnode) ADD_SUBDIRECTORY(vnode) ADD_SUBDIRECTORY(dnode) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 2dd580dda00651d8d7d4d84505147a8f464d954c..fdc0ae909519eba49b48257d65515537016c3f0a 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -294,20 +294,21 @@ typedef struct SResRec { struct STSBuf; typedef struct { - int32_t code; int64_t numOfRows; // num of results in current retrieved int64_t numOfTotal; // num of total results int64_t numOfTotalInCurrentClause; // num of total result in current subclause char * pRsp; - int rspType; - int rspLen; + int32_t rspType; + int32_t rspLen; uint64_t qhandle; int64_t uid; int64_t useconds; int64_t offset; // offset value from vnode during projection query of stable - int row; + int32_t row; int16_t numOfCols; int16_t precision; + bool completed; + int32_t code; int32_t numOfGroups; SResRec * pGroupRec; char * data; diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 689f9715d08b4569817d16daca224c2482c7643c..c7be1b84cefa4271f8dae73c9dd3235b6597f0e0 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -417,6 +417,10 @@ static void count_function(SQLFunctionCtx *pCtx) { numOfElem += 1; } } else { + /* + * when counting on the primary time stamp column and no statistics data is provided, + * simple use the size value + */ numOfElem = pCtx->size; } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 58fc32c824c287d5e426236c41fee34f38526b00..348c0709f17bfcf2bb9c54ddfdd310021f08bd42 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -209,7 +209,6 @@ int tscSendMsgToServer(SSqlObj *pSql) { } void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { - tscTrace("response:%s is received, len:%d error:%s", taosMsg[rpcMsg->msgType], rpcMsg->contLen, tstrerror(rpcMsg->code)); SSqlObj *pSql = (SSqlObj *)rpcMsg->handle; if (pSql == NULL || pSql->signature != pSql) { tscError("%p sql is already released, signature:%p", pSql, pSql->signature); @@ -256,7 +255,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { rpcFreeCont(rpcMsg->pCont); return; } else { - tscTrace("%p it shall renew meter meta, code:%d", pSql, rpcMsg->code); + tscTrace("%p it shall renew meter meta, code:%d", pSql, tstrerror(rpcMsg->code)); pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; pSql->res.code = rpcMsg->code; // keep the previous error code @@ -273,19 +272,19 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { } pSql->retry = 0; - pRes->rspLen = 0; + if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_NETWORK_UNAVAIL; } else { - tscTrace("%p query is cancelled, code:%d", pSql, pRes->code); + tscTrace("%p query is cancelled, code:%d", pSql, tstrerror(pRes->code)); } if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { assert(rpcMsg->msgType == pCmd->msgType + 1); - pRes->code = (int32_t)rpcMsg->code; + pRes->code = rpcMsg->code; pRes->rspType = rpcMsg->msgType; - pRes->rspLen = rpcMsg->contLen; + pRes->rspLen = rpcMsg->contLen; char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); if (tmp == NULL) { @@ -318,19 +317,17 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { tscTrace("%p cmd:%d code:%d, inserted rows:%d, rsp len:%d", pSql, pCmd->command, pRes->code, *(int32_t *)pRes->pRsp, pRes->rspLen); } else { - tscTrace("%p cmd:%d code:%d rsp len:%d", pSql, pCmd->command, pRes->code, pRes->rspLen); + tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen); } } if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); - + if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { int command = pCmd->command; void *taosres = tscKeepConn[command] ? pSql : NULL; - rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows; - - tscTrace("%p Async SQL result:%d res:%p", pSql, rpcMsg->code, taosres); + tscTrace("%p Async SQL result:%s res:%p", pSql, tstrerror(pRes->code), taosres); /* * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj @@ -389,7 +386,7 @@ int tscProcessSql(SSqlObj *pSql) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = NULL; - uint16_t type = 0; + uint16_t type = 0; if (pQueryInfo != NULL) { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -659,14 +656,18 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscError("%p failed to malloc for query msg", pSql); return -1; } - + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - char *pStart = pCmd->payload + tsRpcHeadSize; - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; + + if (pQueryInfo->colList.numOfCols <= 0) { + tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); + return -1; + } + + char *pStart = pCmd->payload + tsRpcHeadSize; SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; @@ -675,11 +676,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { numOfTables = 1; - - pQueryMsg->uid = pTableMeta->uid; - pQueryMsg->numOfTagsCols = 0; - - pQueryMsg->vgId = htonl(pTableMeta->vgId); + pQueryMsg->head.vgId = htonl(pTableMeta->vgId); tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name); } else { // query on super table if (pTableMetaInfo->vnodeIndex < 0) { @@ -697,11 +694,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } tscTrace("%p query on vid:%d, number of tables:%d", pSql, vnodeId, numOfTables); - pQueryMsg->vgId = htons(vnodeId); + pQueryMsg->head.vgId = htons(vnodeId); } pQueryMsg->numOfTables = htonl(numOfTables); - pQueryMsg->numOfTagsCols = htons(pTableMetaInfo->numOfTags); if (pQueryInfo->order.order == TSQL_SO_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->stime); @@ -711,24 +707,15 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->window.ekey = htobe64(pQueryInfo->stime); } - pQueryMsg->order = htons(pQueryInfo->order.order); - pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId); - - pQueryMsg->interpoType = htons(pQueryInfo->interpoType); - - pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); - pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); - - pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols); - - if (pQueryInfo->colList.numOfCols <= 0) { - tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); - return -1; - } - - pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); + pQueryMsg->order = htons(pQueryInfo->order.order); + pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId); + pQueryMsg->interpoType = htons(pQueryInfo->interpoType); + pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); + pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); + pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols); + pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); + pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; - pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); if (pQueryInfo->intervalTime < 0) { tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); @@ -776,7 +763,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->colList[i].colId = htons(pColSchema->colId); pQueryMsg->colList[i].bytes = htons(pColSchema->bytes); - pQueryMsg->colList[i].type = htons(pColSchema->type); + pQueryMsg->colList[i].type = htons(pColSchema->type); pQueryMsg->colList[i].numOfFilters = htons(pCol->numOfFilters); // append the filter information after the basic column information @@ -824,11 +811,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return -1; } - pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); + pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); pSqlFuncExpr->colInfo.colIdx = htons(pExpr->colInfo.colIdx); - pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); + pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); - pSqlFuncExpr->functionId = htons(pExpr->functionId); + pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); pMsg += sizeof(SSqlFuncExprMsg); @@ -866,25 +853,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->colNameLen = htonl(len); // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->vgId), pMsg); - - // only include the required tag column schema. If a tag is not required, it won't be sent to vnode - if (pTableMetaInfo->numOfTags > 0) { - // always transfer tag schema to vnode if exists - SSchema *pTagSchema = tscGetTableTagSchema(pTableMeta); - - for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { - if (pTableMetaInfo->tagColumnIndex[j] == TSDB_TBNAME_COLUMN_INDEX) { - SSchema tbSchema = { - .bytes = TSDB_TABLE_NAME_LEN, .colId = TSDB_TBNAME_COLUMN_INDEX, .type = TSDB_DATA_TYPE_BINARY}; - memcpy(pMsg, &tbSchema, sizeof(SSchema)); - } else { - memcpy(pMsg, &pTagSchema[pTableMetaInfo->tagColumnIndex[j]], sizeof(SSchema)); - } - - pMsg += sizeof(SSchema); - } - } + pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->head.vgId), pMsg); SSqlGroupbyExpr *pGroupbyExpr = &pQueryInfo->groupbyExpr; if (pGroupbyExpr->numOfGroupCols != 0) { @@ -948,8 +917,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = msgLen; pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; - pQueryMsg->contLen = htonl(msgLen); - + pQueryMsg->head.contLen = htonl(msgLen); assert(msgLen + minMsgSize() <= size); return TSDB_CODE_SUCCESS; @@ -2331,9 +2299,10 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { pRes->numOfRows = htonl(pRetrieve->numOfRows); pRes->precision = htons(pRetrieve->precision); - pRes->offset = htobe64(pRetrieve->offset); - pRes->useconds = htobe64(pRetrieve->useconds); - pRes->data = pRetrieve->data; + pRes->offset = htobe64(pRetrieve->offset); + pRes->useconds = htobe64(pRetrieve->useconds); + pRes->completed = (pRetrieve->completed == 1); + pRes->data = pRetrieve->data; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); tscSetResultPointer(pQueryInfo, pRes); @@ -2358,7 +2327,6 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { } pRes->row = 0; - tscTrace("%p numOfRows:%d, offset:%d", pSql, pRes->numOfRows, pRes->offset); return 0; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 63612d0f5f21d6cbacc3ef7118a2f2825624db08..4885cf7cc3cc064f05672be40d3c8303b85de67d 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -653,62 +653,6 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) { return pRes->tsrow; } -TAOS_ROW taos_fetch_row_impl(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - return NULL; - } - - if (pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) { - tscFetchDatablockFromSubquery(pSql); - - if (pRes->code == TSDB_CODE_SUCCESS) { - tscTrace("%p data from all subqueries have been retrieved to client", pSql); - return tscBuildResFromSubqueries(pSql); - } else { - tscTrace("%p retrieve data from subquery failed, code:%d", pSql, pRes->code); - return NULL; - } - - } else if (pRes->row >= pRes->numOfRows) { - /** - * NOT a join query - * - * If the data block of current result set have been consumed already, try fetch next result - * data block from virtual node. - */ - tscResetForNextRetrieve(pRes); - - if (pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - } - - tscProcessSql(pSql); // retrieve data from virtual node - - // if failed to retrieve data from current virtual node, try next one if exists - if (hasMoreVnodesToTry(pSql)) { - tscTryQueryNextVnode(pSql, NULL); - } - - /* - * local reducer has handle this case, - * so no need to add the pRes->numOfRows for super table query - */ - if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC) { - pRes->numOfTotalInCurrentClause += pRes->numOfRows; - } - - if (pRes->numOfRows == 0) { - return NULL; - } - } - - return doSetResultRowData(pSql); -} - static void asyncFetchCallback(void *param, TAOS_RES *tres, int numOfRows) { SSqlObj* pSql = (SSqlObj*) tres; if (numOfRows < 0) { @@ -729,7 +673,10 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT) { + if (pRes->qhandle == 0 || + pRes->completed || + pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + pCmd->command == TSDB_SQL_INSERT) { return NULL; } diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 298eb6110c8dab7ac34a24446d851b37a6b5dace..d85782b463f4ca579c9874ee1050593ff01f40ca 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode sdb taos_static monitor http tsdb) + TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb) #IF (TD_CLUSTER) # TARGET_LINK_LIBRARIES(taosd dcluster) diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index 887845b00ed01820ec11ad641a732904145559ab..3419128c72647c5a88dd62ab1ff51f80f407e26c 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -93,8 +93,8 @@ void dnodeRead(SRpcMsg *pMsg) { while (leftLen > 0) { SMsgHead *pHead = (SMsgHead *) pCont; - pHead->vgId = 1;//htonl(pHead->vgId); - pHead->contLen = pMsg->contLen; //htonl(pHead->contLen); + pHead->vgId = htonl(pHead->vgId); + pHead->contLen = htonl(pHead->contLen); void *pVnode = dnodeGetVnode(pHead->vgId); if (pVnode == NULL) { @@ -104,12 +104,13 @@ void dnodeRead(SRpcMsg *pMsg) { } // put message into queue - SReadMsg readMsg; - readMsg.rpcMsg = *pMsg; - readMsg.pCont = pCont; - readMsg.contLen = pHead->contLen; - readMsg.pRpcContext = pRpcContext; - readMsg.pVnode = pVnode; + SReadMsg readMsg = { + .rpcMsg = *pMsg, + .pCont = pCont, + .contLen = pHead->contLen, + .pRpcContext = pRpcContext, + .pVnode = pVnode, + }; taos_queue queue = dnodeGetVnodeRworker(pVnode); taosWriteQitem(queue, &readMsg); @@ -177,8 +178,6 @@ static void *dnodeProcessReadQueue(void *param) { } else { terrno = TSDB_CODE_MSG_NOT_PROCESSED; } - - dnodeProcessReadResult(&readMsg); } return NULL; @@ -252,57 +251,42 @@ static void dnodeProcessQueryMsg(SReadMsg *pMsg) { qTableQuery(pQInfo); } +static int32_t c = 0; static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { SRetrieveTableMsg *pRetrieve = pMsg->pCont; - void *pQInfo = htobe64(pRetrieve->qhandle); + void *pQInfo = (void*) htobe64(pRetrieve->qhandle); dTrace("QInfo:%p vgId:%d, retrieve msg is received", pQInfo, pRetrieve->header.vgId); - + if ((++c)%2 == 0) { + int32_t k = 1; + } int32_t rowSize = 0; int32_t numOfRows = 0; int32_t contLen = 0; - SRpcMsg rpcRsp = {0}; + SRetrieveTableRsp *pRsp = NULL; int32_t code = qRetrieveQueryResultInfo(pQInfo, &numOfRows, &rowSize); if (code != TSDB_CODE_SUCCESS) { contLen = sizeof(SRetrieveTableRsp); - - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); - pRsp->numOfRows = 0; - pRsp->precision = 0; - pRsp->offset = 0; - pRsp->useconds = 0; - - rpcRsp = (SRpcMsg) { - .handle = pMsg->rpcMsg.handle, - .pCont = pRsp, - .contLen = contLen, - .code = code, - .msgType = 0 - }; - - //todo free qinfo + + pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); + memset(pRsp, 0, sizeof(SRetrieveTableRsp)); } else { - contLen = 100; - - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); - pRsp->numOfRows = htonl(1); - pRsp->precision = htons(0); - pRsp->offset = htobe64(0); - pRsp->useconds = htobe64(0); - - // todo set the data - *(int64_t*) pRsp->data = 1000; - - rpcRsp = (SRpcMsg) { - .handle = pMsg->rpcMsg.handle, - .pCont = pRsp, - .contLen = contLen, - .code = code, - .msgType = 0 - }; + // todo check code and handle error in build result set + code = qDumpRetrieveResult(pQInfo, &pRsp, &contLen); } + SRpcMsg rpcRsp = (SRpcMsg) { + .handle = pMsg->rpcMsg.handle, + .pCont = pRsp, + .contLen = contLen, + .code = code, + .msgType = 0 + }; + rpcSendResponse(&rpcRsp); + + //todo merge result should be done here + //dnodeProcessReadResult(&readMsg); } diff --git a/src/inc/mnode.h b/src/inc/mnode.h index be152fbf3cbc6000cb40affbe99ea07e3c62b05e..792a8ec1cce02ed0b00ef5d44b32624ccfb9ca0e 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -25,7 +25,6 @@ extern "C" { #include "taosdef.h" #include "taosmsg.h" #include "taoserror.h" -#include "sdb.h" #include "tglobalcfg.h" #include "thash.h" #include "tidpool.h" @@ -54,8 +53,8 @@ typedef struct { int8_t numOfMnodes; int32_t numOfDnodes; char mnodeName[TSDB_DNODE_NAME_LEN + 1]; - char reserved[7]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; int syncFd; void *hbTimer; void *pSync; @@ -79,8 +78,8 @@ typedef struct { float lbScore; // calc in balance function int32_t customScore; // config by user char dnodeName[TSDB_DNODE_NAME_LEN + 1]; - char reserved[7]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; SVnodeLoad vload[TSDB_MAX_VNODES]; int32_t status; uint32_t lastReboot; // time stamp for last reboot @@ -121,7 +120,7 @@ typedef struct SSuperTableObj { int32_t sversion; int32_t numOfColumns; int32_t numOfTags; - int8_t reserved[5]; + int8_t reserved[15]; int8_t updateEnd[1]; int32_t numOfTables; int16_t nextColId; @@ -153,26 +152,29 @@ typedef struct { int32_t sversion; int32_t numOfColumns; int32_t sqlLen; - int8_t reserved[3]; + int8_t reserved[7]; int8_t updateEnd[1]; char* sql; //null-terminated string int16_t nextColId; SSchema* schema; } SNormalTableObj; +struct _db_obj; + typedef struct _vg_obj { uint32_t vgId; char dbName[TSDB_DB_NAME_LEN + 1]; int64_t createdTime; SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT]; int32_t numOfVnodes; - int32_t numOfTables; int32_t lbIp; int32_t lbTime; int8_t lbStatus; int8_t reserved[14]; int8_t updateEnd[1]; struct _vg_obj *prev, *next; + struct _db_obj *pDb; + int32_t numOfTables; void * idPool; STableInfo ** tableList; } SVgObj; @@ -182,8 +184,8 @@ typedef struct _db_obj { int8_t dirty; int64_t createdTime; SDbCfg cfg; - char reserved[15]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; struct _db_obj *prev, *next; int32_t numOfVgroups; int32_t numOfTables; @@ -201,7 +203,7 @@ typedef struct _user_obj { int64_t createdTime; int8_t superAuth; int8_t writeAuth; - int8_t reserved[16]; + int8_t reserved[13]; int8_t updateEnd[1]; struct _user_obj *prev, *next; struct _acctObj * pAcct; diff --git a/src/inc/sdb.h b/src/inc/sdb.h deleted file mode 100644 index 4b4de1ac4b9b2575493f289f98660dbbb6704911..0000000000000000000000000000000000000000 --- a/src/inc/sdb.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SDB_H -#define TDENGINE_SDB_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "taosmsg.h" -#include "taosdef.h" - -extern uint16_t tsMgmtMgmtPort; -extern uint16_t tsMgmtSyncPort; -extern int tsMgmtPeerHBTimer; // seconds -extern char * sdbStatusStr[]; -extern char * sdbRoleStr[]; -extern int sdbMaster; -extern SRpcIpSet *pSdbIpList; -extern SRpcIpSet *pSdbPublicIpList; - -extern void (*sdbWorkAsMasterCallback)(); // this function pointer will be set by taosd - -enum _keytype { - SDB_KEYTYPE_STRING, SDB_KEYTYPE_UINT32, SDB_KEYTYPE_AUTO, SDB_KEYTYPE_RECYCLE, SDB_KEYTYPE_MAX -}; - -#define SDB_ROLE_UNAPPROVED 0 -#define SDB_ROLE_UNDECIDED 1 -#define SDB_ROLE_MASTER 2 -#define SDB_ROLE_SLAVE 3 - -#define SDB_STATUS_OFFLINE 0 -#define SDB_STATUS_UNSYNCED 1 -#define SDB_STATUS_SYNCING 2 -#define SDB_STATUS_SERVING 3 -#define SDB_STATUS_DELETED 4 - -enum _sdbaction { - SDB_TYPE_INSERT, - SDB_TYPE_DELETE, - SDB_TYPE_UPDATE, - SDB_TYPE_DECODE, - SDB_TYPE_ENCODE, - SDB_TYPE_BEFORE_BATCH_UPDATE, - SDB_TYPE_BATCH_UPDATE, - SDB_TYPE_AFTER_BATCH_UPDATE, - SDB_TYPE_RESET, - SDB_TYPE_DESTROY, - SDB_MAX_ACTION_TYPES -}; - -#define SDB_MAX_PEERS 4 -typedef struct { - uint32_t ip; - uint32_t publicIp; - char ipstr[20]; - char zone[12]; - char role; - int64_t createdTime; - uint64_t dbVersion; - int64_t lostTime; - char status; - char numOfMnodes; - int numOfDnodes; - char updateEnd[1]; - - // internal - int syncFd; - void *hbTimer; - void *pSync; -} SSdbPeer; - -extern SSdbPeer *sdbPeer[]; -#define sdbInited (sdbPeer[0]) -#define sdbStatus (sdbPeer[0]->status) - -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int, int *)); - -void *sdbGetRow(void *handle, void *key); - -int64_t sdbInsertRow(void *handle, void *row, int rowSize); - -int sdbDeleteRow(void *handle, void *key); - -int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated); - -void *sdbFetchRow(void *handle, void *pNode, void **ppRow); - -int sdbBatchUpdateRow(void *handle, void *row, int rowSize); - -int64_t sdbGetId(void *handle); - -int64_t sdbGetNumOfRows(void *handle); - -void sdbSaveSnapShot(void *handle); - -void sdbCloseTable(void *handle); - -int sdbRemovePeerByIp(uint32_t ip); - -int sdbInitPeers(char *directory); - -void sdbCleanUpPeers(); - -int64_t sdbGetVersion(); - -int32_t sdbGetRunStatus(); - -#define TSDB_MAX_NORMAL_TABLES 10000 -#define TSDB_MAX_SUPER_TABLES 1000 - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_SDB_H diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 5da8aa191b5e1ce5072a80599a0c64b3f44d9f15..570e2e1acd0391f2f59f56b9b68b52e38310da14 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -309,6 +309,16 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_SESSIONS_PER_VNODE (300) #define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES) +#define TSDB_MAX_MNODES 5 +#define TSDB_MAX_DNODES 10 +#define TSDB_MAX_ACCOUNTS 10 +#define TSDB_MAX_USERS 20 +#define TSDB_MAX_DBS 100 +#define TSDB_MAX_VGROUPS 1000 +#define TSDB_MAX_SUPER_TABLES 100 +#define TSDB_MAX_NORMAL_TABLES 1000 +#define TSDB_MAX_CHILD_TABLES 100000 + enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index cc17df9bec97d30b4efe3b6bd5ae9a0cb1d6afff..2fcc91a7bb84d98ff9ae4e898dc77472b332b008 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -20,122 +20,122 @@ extern "C" { #endif -#include #include +#include #include "taosdef.h" #include "taoserror.h" #include "trpc.h" // message type -#define TSDB_MSG_TYPE_REG 1 -#define TSDB_MSG_TYPE_REG_RSP 2 -#define TSDB_MSG_TYPE_SUBMIT 3 -#define TSDB_MSG_TYPE_SUBMIT_RSP 4 -#define TSDB_MSG_TYPE_QUERY 5 -#define TSDB_MSG_TYPE_QUERY_RSP 6 -#define TSDB_MSG_TYPE_RETRIEVE 7 -#define TSDB_MSG_TYPE_RETRIEVE_RSP 8 +#define TSDB_MSG_TYPE_REG 1 +#define TSDB_MSG_TYPE_REG_RSP 2 +#define TSDB_MSG_TYPE_SUBMIT 3 +#define TSDB_MSG_TYPE_SUBMIT_RSP 4 +#define TSDB_MSG_TYPE_QUERY 5 +#define TSDB_MSG_TYPE_QUERY_RSP 6 +#define TSDB_MSG_TYPE_RETRIEVE 7 +#define TSDB_MSG_TYPE_RETRIEVE_RSP 8 // message from mnode to dnode -#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9 -#define TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP 10 -#define TSDB_MSG_TYPE_MD_DROP_TABLE 11 -#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12 -#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13 -#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14 -#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15 -#define TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP 16 -#define TSDB_MSG_TYPE_MD_DROP_VNODE 17 -#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18 -#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19 -#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20 -#define TSDB_MSG_TYPE_MD_DROP_STABLE 21 -#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22 -#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23 -#define TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP 24 -#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25 -#define TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP 26 +#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9 +#define TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP 10 +#define TSDB_MSG_TYPE_MD_DROP_TABLE 11 +#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12 +#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13 +#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14 +#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15 +#define TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP 16 +#define TSDB_MSG_TYPE_MD_DROP_VNODE 17 +#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18 +#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19 +#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20 +#define TSDB_MSG_TYPE_MD_DROP_STABLE 21 +#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22 +#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23 +#define TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP 24 +#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25 +#define TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP 26 // message from client to mnode -#define TSDB_MSG_TYPE_CM_CONNECT 31 -#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32 -#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33 -#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34 -#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35 -#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36 -#define TSDB_MSG_TYPE_CM_DROP_ACCT 37 -#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38 -#define TSDB_MSG_TYPE_CM_CREATE_USER 39 -#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40 -#define TSDB_MSG_TYPE_CM_ALTER_USER 41 -#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42 -#define TSDB_MSG_TYPE_CM_DROP_USER 43 -#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44 -#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45 -#define TSDB_MSG_TYPE_CM_CREATE_DNODE_RSP 46 -#define TSDB_MSG_TYPE_CM_DROP_DNODE 47 -#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48 -#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE -#define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP -#define TSDB_MSG_TYPE_CM_CREATE_DB 49 -#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50 -#define TSDB_MSG_TYPE_CM_DROP_DB 51 -#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52 -#define TSDB_MSG_TYPE_CM_USE_DB 53 -#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54 -#define TSDB_MSG_TYPE_CM_ALTER_DB 55 -#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56 -#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57 -#define TSDB_MSG_TYPE_CM_CREATE_TABLE_RSP 58 -#define TSDB_MSG_TYPE_CM_DROP_TABLE 59 -#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60 -#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61 -#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 -#define TSDB_MSG_TYPE_CM_TABLE_META 63 -#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 -#define TSDB_MSG_TYPE_CM_STABLE_META 65 -#define TSDB_MSG_TYPE_CM_STABLE_META_RSP 66 -#define TSDB_MSG_TYPE_CM_TABLES_META 67 -#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 -#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 -#define TSDB_MSG_TYPE_CM_ALTER_STREAM_RSP 70 -#define TSDB_MSG_TYPE_CM_SHOW 71 -#define TSDB_MSG_TYPE_CM_SHOW_RSP 72 -#define TSDB_MSG_TYPE_CM_KILL_QUERY 73 -#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74 -#define TSDB_MSG_TYPE_CM_KILL_STREAM 75 -#define TSDB_MSG_TYPE_CM_KILL_STREAM_RSP 76 -#define TSDB_MSG_TYPE_CM_KILL_CONN 77 -#define TSDB_MSG_TYPE_CM_KILL_CONN_RSP 78 -#define TSDB_MSG_TYPE_CM_HEARTBEAT 79 -#define TSDB_MSG_TYPE_CM_HEARTBEAT_RSP 80 +#define TSDB_MSG_TYPE_CM_CONNECT 31 +#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32 +#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33 +#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34 +#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35 +#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36 +#define TSDB_MSG_TYPE_CM_DROP_ACCT 37 +#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38 +#define TSDB_MSG_TYPE_CM_CREATE_USER 39 +#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40 +#define TSDB_MSG_TYPE_CM_ALTER_USER 41 +#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42 +#define TSDB_MSG_TYPE_CM_DROP_USER 43 +#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44 +#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45 +#define TSDB_MSG_TYPE_CM_CREATE_DNODE_RSP 46 +#define TSDB_MSG_TYPE_CM_DROP_DNODE 47 +#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48 +#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE +#define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP +#define TSDB_MSG_TYPE_CM_CREATE_DB 49 +#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50 +#define TSDB_MSG_TYPE_CM_DROP_DB 51 +#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52 +#define TSDB_MSG_TYPE_CM_USE_DB 53 +#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54 +#define TSDB_MSG_TYPE_CM_ALTER_DB 55 +#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56 +#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57 +#define TSDB_MSG_TYPE_CM_CREATE_TABLE_RSP 58 +#define TSDB_MSG_TYPE_CM_DROP_TABLE 59 +#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60 +#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61 +#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 +#define TSDB_MSG_TYPE_CM_TABLE_META 63 +#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 +#define TSDB_MSG_TYPE_CM_STABLE_META 65 +#define TSDB_MSG_TYPE_CM_STABLE_META_RSP 66 +#define TSDB_MSG_TYPE_CM_TABLES_META 67 +#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 +#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 +#define TSDB_MSG_TYPE_CM_ALTER_STREAM_RSP 70 +#define TSDB_MSG_TYPE_CM_SHOW 71 +#define TSDB_MSG_TYPE_CM_SHOW_RSP 72 +#define TSDB_MSG_TYPE_CM_KILL_QUERY 73 +#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74 +#define TSDB_MSG_TYPE_CM_KILL_STREAM 75 +#define TSDB_MSG_TYPE_CM_KILL_STREAM_RSP 76 +#define TSDB_MSG_TYPE_CM_KILL_CONN 77 +#define TSDB_MSG_TYPE_CM_KILL_CONN_RSP 78 +#define TSDB_MSG_TYPE_CM_HEARTBEAT 79 +#define TSDB_MSG_TYPE_CM_HEARTBEAT_RSP 80 // message from dnode to mnode -#define TSDB_MSG_TYPE_DM_CONFIG_TABLE 91 -#define TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP 92 -#define TSDB_MSG_TYPE_DM_CONFIG_VNODE 93 -#define TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP 94 -#define TSDB_MSG_TYPE_DM_STATUS 95 -#define TSDB_MSG_TYPE_DM_STATUS_RSP 96 -#define TSDB_MSG_TYPE_DM_GRANT 97 -#define TSDB_MSG_TYPE_DM_GRANT_RSP 98 - -#define TSDB_MSG_TYPE_SDB_SYNC 101 -#define TSDB_MSG_TYPE_SDB_SYNC_RSP 102 -#define TSDB_MSG_TYPE_SDB_FORWARD 103 -#define TSDB_MSG_TYPE_SDB_FORWARD_RSP 104 - -#define TSDB_MSG_TYPE_MAX 105 +#define TSDB_MSG_TYPE_DM_CONFIG_TABLE 91 +#define TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP 92 +#define TSDB_MSG_TYPE_DM_CONFIG_VNODE 93 +#define TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP 94 +#define TSDB_MSG_TYPE_DM_STATUS 95 +#define TSDB_MSG_TYPE_DM_STATUS_RSP 96 +#define TSDB_MSG_TYPE_DM_GRANT 97 +#define TSDB_MSG_TYPE_DM_GRANT_RSP 98 + +#define TSDB_MSG_TYPE_SDB_SYNC 101 +#define TSDB_MSG_TYPE_SDB_SYNC_RSP 102 +#define TSDB_MSG_TYPE_SDB_FORWARD 103 +#define TSDB_MSG_TYPE_SDB_FORWARD_RSP 104 + +#define TSDB_MSG_TYPE_MAX 105 // IE type -#define TSDB_IE_TYPE_SEC 1 -#define TSDB_IE_TYPE_META 2 -#define TSDB_IE_TYPE_MGMT_IP 3 -#define TSDB_IE_TYPE_DNODE_CFG 4 -#define TSDB_IE_TYPE_NEW_VERSION 5 -#define TSDB_IE_TYPE_DNODE_EXT 6 -#define TSDB_IE_TYPE_DNODE_STATE 7 +#define TSDB_IE_TYPE_SEC 1 +#define TSDB_IE_TYPE_META 2 +#define TSDB_IE_TYPE_MGMT_IP 3 +#define TSDB_IE_TYPE_DNODE_CFG 4 +#define TSDB_IE_TYPE_NEW_VERSION 5 +#define TSDB_IE_TYPE_DNODE_EXT 6 +#define TSDB_IE_TYPE_DNODE_STATE 7 enum _mgmt_table { TSDB_MGMT_TABLE_ACCT, @@ -157,38 +157,38 @@ enum _mgmt_table { TSDB_MGMT_TABLE_MAX, }; -#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 -#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 -#define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3 -#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 +#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 +#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 +#define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3 +#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 -#define TSDB_ALTER_TABLE_ADD_COLUMN 5 -#define TSDB_ALTER_TABLE_DROP_COLUMN 6 +#define TSDB_ALTER_TABLE_ADD_COLUMN 5 +#define TSDB_ALTER_TABLE_DROP_COLUMN 6 -#define TSDB_INTERPO_NONE 0 -#define TSDB_INTERPO_NULL 1 -#define TSDB_INTERPO_SET_VALUE 2 -#define TSDB_INTERPO_LINEAR 3 -#define TSDB_INTERPO_PREV 4 +#define TSDB_INTERPO_NONE 0 +#define TSDB_INTERPO_NULL 1 +#define TSDB_INTERPO_SET_VALUE 2 +#define TSDB_INTERPO_LINEAR 3 +#define TSDB_INTERPO_PREV 4 -#define TSDB_ALTER_USER_PASSWD 0x1 -#define TSDB_ALTER_USER_PRIVILEGES 0x2 +#define TSDB_ALTER_USER_PASSWD 0x1 +#define TSDB_ALTER_USER_PRIVILEGES 0x2 -#define TSDB_KILL_MSG_LEN 30 +#define TSDB_KILL_MSG_LEN 30 -#define TSDB_VN_READ_ACCCESS ((char)0x1) +#define TSDB_VN_READ_ACCCESS ((char)0x1) #define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u -#define TSDB_COL_TAG 0x1u -#define TSDB_COL_JOIN 0x2u +#define TSDB_COL_NORMAL 0x0u +#define TSDB_COL_TAG 0x1u +#define TSDB_COL_JOIN 0x2u extern char *taosMsg[]; #pragma pack(push, 1) -//typedef struct { +// typedef struct { // int32_t vnode; // int32_t sid; // int32_t sversion; @@ -206,7 +206,7 @@ typedef struct SMsgHead { int32_t vgId; } SMsgHead; -//typedef struct { +// typedef struct { // SMsgDesc desc; // SMsgHead header; // int16_t import; @@ -216,37 +216,37 @@ typedef struct SMsgHead { // Submit message for one table typedef struct SSubmitBlk { - int64_t uid; // table unique id - int32_t tid; // table id - int32_t padding; // TODO just for padding here - int32_t sversion; // data schema version - int32_t len; // data part length, not including the SSubmitBlk head - int16_t numOfRows; // total number of rows in current submit block - char data[]; + int64_t uid; // table unique id + int32_t tid; // table id + int32_t padding; // TODO just for padding here + int32_t sversion; // data schema version + int32_t len; // data part length, not including the SSubmitBlk head + int16_t numOfRows; // total number of rows in current submit block + char data[]; } SSubmitBlk; // Submit message for this TSDB typedef struct SSubmitMsg { SMsgHead header; int32_t length; - int32_t compressed:2; - int32_t numOfBlocks:30; + int32_t compressed : 2; + int32_t numOfBlocks : 30; SSubmitBlk blocks[]; } SSubmitMsg; typedef struct { - int32_t index; // index of failed block in submit blocks - int32_t vnode; // vnode index of failed block - int32_t sid; // table index of failed block - int32_t code; // errorcode while write data to vnode, such as not created, dropped, no space, invalid table + int32_t index; // index of failed block in submit blocks + int32_t vnode; // vnode index of failed block + int32_t sid; // table index of failed block + int32_t code; // errorcode while write data to vnode, such as not created, dropped, no space, invalid table } SShellSubmitRspBlock; typedef struct { - int32_t code; // 0-success, > 0 error code - int32_t numOfRows; // number of records the client is trying to write - int32_t affectedRows; // number of records actually written - int32_t failedRows; // number of failed records (exclude duplicate records) - int32_t numOfFailedBlocks; + int32_t code; // 0-success, > 0 error code + int32_t numOfRows; // number of records the client is trying to write + int32_t affectedRows; // number of records actually written + int32_t failedRows; // number of failed records (exclude duplicate records) + int32_t numOfFailedBlocks; SShellSubmitRspBlock failedBlocks[]; } SShellSubmitRspMsg; @@ -258,38 +258,38 @@ typedef struct SSchema { } SSchema; typedef struct { - int32_t vnode; //the index of vnode + int32_t vnode; // the index of vnode uint32_t ip; } SVnodeDesc; typedef struct { - int32_t contLen; - int32_t vgId; - int8_t tableType; - int16_t numOfColumns; - int16_t numOfTags; - int32_t sid; - int32_t sversion; - int32_t tagDataLen; - int32_t sqlDataLen; - uint64_t uid; - uint64_t superTableUid; - uint64_t createdTime; - char tableId[TSDB_TABLE_ID_LEN]; - char superTableId[TSDB_TABLE_ID_LEN]; - char data[]; + int32_t contLen; + int32_t vgId; + int8_t tableType; + int16_t numOfColumns; + int16_t numOfTags; + int32_t sid; + int32_t sversion; + int32_t tagDataLen; + int32_t sqlDataLen; + uint64_t uid; + uint64_t superTableUid; + uint64_t createdTime; + char tableId[TSDB_TABLE_ID_LEN]; + char superTableId[TSDB_TABLE_ID_LEN]; + char data[]; } SMDCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - char db[TSDB_DB_NAME_LEN]; - int8_t igExists; - int16_t numOfTags; - int16_t numOfColumns; - int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string - int32_t contLen; - int8_t reserved[16]; - char schema[]; + char tableId[TSDB_TABLE_ID_LEN]; + char db[TSDB_DB_NAME_LEN]; + int8_t igExists; + int16_t numOfTags; + int16_t numOfColumns; + int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string + int32_t contLen; + int8_t reserved[16]; + char schema[]; } SCMCreateTableMsg; typedef struct { @@ -331,7 +331,7 @@ typedef struct { int64_t maxQueryTime; // In unit of hour int64_t maxInbound; int64_t maxOutbound; - int8_t accessState; // Configured only by command + int8_t accessState; // Configured only by command } SAcctCfg; typedef struct { @@ -356,11 +356,11 @@ typedef struct { } SMgmtHead; typedef struct { - int32_t contLen; - int32_t vgId; - int32_t sid; - uint64_t uid; - char tableId[TSDB_TABLE_ID_LEN + 1]; + int32_t contLen; + int32_t vgId; + int32_t sid; + uint64_t uid; + char tableId[TSDB_TABLE_ID_LEN + 1]; } SMDDropTableMsg; typedef struct { @@ -371,7 +371,7 @@ typedef struct { } SMDDropSTableMsg; typedef struct { - int32_t vgId; + int32_t vgId; } SMDDropVnodeMsg; typedef struct SColIndexEx { @@ -386,7 +386,7 @@ typedef struct SColIndexEx { */ int16_t colIdx; int16_t colIdxInBuf; - uint16_t flag; // denote if it is a tag or not + uint16_t flag; // denote if it is a tag or not char name[TSDB_COL_NAME_LEN]; } SColIndexEx; @@ -458,7 +458,7 @@ typedef struct SColumnInfo { typedef struct STableIdInfo { int32_t sid; int64_t uid; - TSKEY key; // last accessed ts, for subscription + TSKEY key; // last accessed ts, for subscription } STableIdInfo; typedef struct STimeWindow { @@ -472,47 +472,30 @@ typedef struct STimeWindow { * the outputCols will be 3 while the numOfCols is 1. */ typedef struct { - int32_t contLen; // msg header - int16_t vgId; - - int32_t numOfTables; - uint64_t uid; + SMsgHead head; STimeWindow window; - - int16_t order; - int16_t orderColId; - - int16_t numOfCols; // the number of columns will be load from vnode - char slidingTimeUnit; // time interval type, for revisement of interval(1d) - - int64_t intervalTime; // time interval for aggregation, in million second - int64_t slidingTime; // value for sliding window - - // tag schema, used to parse tag information in pSidExtInfo - uint64_t pTagSchema; - - int16_t numOfTagsCols; // required number of tags - int16_t tagLength; // tag length in current query - - int16_t numOfGroupCols; // num of group by columns - int16_t orderByIdx; - int16_t orderType; // used in group by xx order by xxx - uint64_t groupbyTagIds; - - int64_t limit; - int64_t offset; - - int16_t queryType; // denote another query process - int16_t numOfOutputCols; // final output columns numbers - - int16_t interpoType; // interpolate type - uint64_t defaultVal; // default value array list - - int32_t colNameLen; - int64_t colNameList; - - int64_t pSqlFuncExprs; - + int32_t numOfTables; + int16_t order; + int16_t orderColId; + int16_t numOfCols; // the number of columns will be load from vnode + int64_t intervalTime; // time interval for aggregation, in million second + int64_t intervalOffset; // start offset for interval query + int64_t slidingTime; // value for sliding window + char slidingTimeUnit; // time interval type, for revisement of interval(1d) + int16_t tagLength; // tag length in current query + int16_t numOfGroupCols; // num of group by columns + int16_t orderByIdx; + int16_t orderType; // used in group by xx order by xxx + uint64_t groupbyTagIds; + int64_t limit; + int64_t offset; + int16_t queryType; // denote another query process + int16_t numOfOutputCols; // final output columns numbers + int16_t interpoType; // interpolate type + uint64_t defaultVal; // default value array list + + int32_t colNameLen; + int64_t colNameList; int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed int32_t tsLen; // total length of ts comp block int32_t tsNumOfBlocks; // ts comp block numbers @@ -533,9 +516,9 @@ typedef struct { typedef struct SRetrieveTableRsp { int32_t numOfRows; - int8_t completed; // all results are returned to client + int8_t completed; // all results are returned to client int16_t precision; - int64_t offset; // updated offset value for multi-vnode projection query + int64_t offset; // updated offset value for multi-vnode projection query int64_t useconds; char data[]; } SRetrieveTableRsp; @@ -615,11 +598,11 @@ typedef struct { char dnodeName[TSDB_DNODE_NAME_LEN]; uint32_t privateIp; uint32_t publicIp; - uint32_t lastReboot; // time stamp for last reboot - uint16_t numOfTotalVnodes; // from config file + uint32_t lastReboot; // time stamp for last reboot + uint16_t numOfTotalVnodes; // from config file uint16_t openVnodes; uint16_t numOfCores; - float diskAvailable; // GB + float diskAvailable; // GB uint8_t alternativeRole; uint8_t reserve[15]; SVnodeLoad load[]; @@ -648,7 +631,7 @@ typedef struct { } SCMMultiTableInfoMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; + char tableId[TSDB_TABLE_ID_LEN + 1]; } SCMSuperTableInfoMsg; typedef struct { @@ -707,18 +690,18 @@ typedef struct STableMetaMsg { uint8_t tableType; int16_t numOfColumns; int16_t sversion; - - int8_t numOfVpeers; + + int8_t numOfVpeers; SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT]; - int32_t sid; - int32_t vgId; - uint64_t uid; - SSchema schema[]; + int32_t sid; + int32_t vgId; + uint64_t uid; + SSchema schema[]; } STableMetaMsg; typedef struct SMultiTableMeta { - int32_t numOfTables; - int32_t contLen; + int32_t numOfTables; + int32_t contLen; STableMetaMsg metas[]; } SMultiTableMeta; @@ -756,7 +739,7 @@ typedef struct { typedef struct { uint32_t dnode; - int32_t vnode; + int32_t vnode; } SDMConfigVnodeMsg; typedef struct { @@ -783,13 +766,13 @@ typedef struct { } SStreamDesc; typedef struct { - int32_t numOfQueries; - SQueryDesc qdesc[]; + int32_t numOfQueries; + SQueryDesc qdesc[]; } SQqueryList; typedef struct { - int32_t numOfStreams; - SStreamDesc sdesc[]; + int32_t numOfStreams; + SStreamDesc sdesc[]; } SStreamList; typedef struct { diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 66e8cf73988ab25db7544b9a52215d2279630c63..386c8a92f781670f38580cbc794e4e0d3044fce4 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -2,5 +2,5 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(shell) -ADD_SUBDIRECTORY(taosdemo) -ADD_SUBDIRECTORY(taosdump) +#ADD_SUBDIRECTORY(taosdemo) +#ADD_SUBDIRECTORY(taosdump) diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 5bf4cfd604c81f14a95129bcd6887d2ca2904b4f..bacd7497ad5832b963db7d188550e4ce560829c7 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(mnode ${SRC}) - TARGET_LINK_LIBRARIES(mnode trpc tutil sdb pthread) + TARGET_LINK_LIBRARIES(mnode trpc tutil pthread) IF (TD_CLUSTER) TARGET_LINK_LIBRARIES(mnode) diff --git a/src/mnode/inc/mgmtChildTable.h b/src/mnode/inc/mgmtChildTable.h index b16dd58f67402e84c2c611282fded15abea88e97..9252a7d485ae02055de81835f2c8c0a0d0da5cbe 100644 --- a/src/mnode/inc/mgmtChildTable.h +++ b/src/mnode/inc/mgmtChildTable.h @@ -39,6 +39,7 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); void mgmtDropAllChildTables(SDbObj *pDropDb); +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtDb.h b/src/mnode/inc/mgmtDb.h index 32bb9d9ec63f386200f85f00d2c5d92c07b04273..f2049f836ceb20a775b25927c91ef040cda78338 100644 --- a/src/mnode/inc/mgmtDb.h +++ b/src/mnode/inc/mgmtDb.h @@ -22,11 +22,11 @@ extern "C" { #include "mnode.h" -int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup); +void mgmtAddVgroupIntoDb(SVgObj *pVgroup); +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup); +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup); +void mgmtMoveVgroupToTail(SVgObj *pVgroup); +void mgmtMoveVgroupToHead(SVgObj *pVgroup); int32_t mgmtInitDbs(); void mgmtCleanUpDbs(); diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index d768d2dd7cd7ca517a02d5dcce88545b178e2851..729d31544f4c49f729e8c6d958a5a038661e225b 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -20,9 +20,15 @@ extern "C" { #endif -bool mgmtCheckRedirect(void *handle); +int32_t mgmtInitMnodes(); +void mgmtCleanupMnodes(); + +bool mgmtInServerStatus(); +bool mgmtIsMaster(); -void mgmtGetMnodeIpList(SRpcIpSet *ipSet); +bool mgmtCheckRedirect(void *handle); +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp); int32_t mgmtRemoveMnode(uint32_t privateIp); diff --git a/src/mnode/inc/mgmtProfile.h b/src/mnode/inc/mgmtProfile.h index 7f9fd9622c7b782e1d796fc36f6340198702934b..8c2e073c2d5981d54f6ca8fc41144bb03f8467b2 100644 --- a/src/mnode/inc/mgmtProfile.h +++ b/src/mnode/inc/mgmtProfile.h @@ -28,6 +28,8 @@ bool mgmtCheckQhandle(uint64_t qhandle); void mgmtSaveQhandle(void *qhandle); void mgmtFreeQhandle(void *qhandle); +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h new file mode 100644 index 0000000000000000000000000000000000000000..ccddc11e04fc32cf2ee8cd0b503c51a993025d4d --- /dev/null +++ b/src/mnode/inc/mgmtSdb.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_MNODE_SDB_H +#define TDENGINE_MNODE_SDB_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + SDB_KEY_TYPE_STRING, + SDB_KEY_TYPE_AUTO +} ESdbKeyType; + +typedef enum { + SDB_OPER_TYPE_GLOBAL, + SDB_OPER_TYPE_LOCAL +} ESdbOperType; + +typedef struct { + ESdbOperType type; + void * table; + void * pObj; + int64_t version; + int32_t maxRowSize; + int32_t rowSize; + void * rowData; +} SSdbOperDesc; + +typedef struct { + char *tableName; + int32_t hashSessions; + int32_t maxRowSize; + ESdbKeyType keyType; + int32_t (*insertFp)(SSdbOperDesc *pOper); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pDesc); + int32_t (*destroyFp)(SSdbOperDesc *pDesc); +} SSdbTableDesc; + +void *sdbOpenTable(SSdbTableDesc *desc); +void sdbCloseTable(void *handle); + +int32_t sdbInsertRow(SSdbOperDesc *pOper); +int32_t sdbDeleteRow(SSdbOperDesc *pOper); +int32_t sdbUpdateRow(SSdbOperDesc *pOper); + +void *sdbGetRow(void *handle, void *key); +void *sdbFetchRow(void *handle, void *pNode, void **ppRow); +int64_t sdbGetNumOfRows(void *handle); +uint64_t sdbGetVersion(); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index b7c68b5f8062240fd1f7c71f1e789d3a31f08630..3379a93f28f0dc8933fcff3cf51bc587c01cc444 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -28,13 +28,12 @@ int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode); +void mgmtDropAllVgroups(SDbObj *pDropDb); void mgmtCreateVgroup(SQueuedMsg *pMsg); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); void mgmtUpdateVgroup(SVgObj *pVgroup); -void mgmtUpdateVgroupIp(SDnodeObj *pDnode); - -void mgmtSetVgroupIdPool(); +void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle); SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb); void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable); diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c index b1ee72386ef9bc36635c982685e7885bd1ab5f93..0075f53319cf63f92457c02204d4d1ab098706c5 100644 --- a/src/mnode/src/mgmtAcct.c +++ b/src/mnode/src/mgmtAcct.c @@ -85,8 +85,10 @@ int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) { pUser->prev->next = pUser->next; } - if (pUser->next) pUser->next->prev = pUser->prev; - + if (pUser->next) { + pUser->next->prev = pUser->prev; + } + if (pUser->prev == NULL) { pAcct->pUser = pUser->next; } diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index d5f0e7c85378d2b775a323a3d1a1bad783088a3f..9076681caa3dfc5e28f85f4c36ca96ff01978165 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -25,79 +25,47 @@ #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDb.h" +#include "mgmtDClient.h" #include "mgmtGrant.h" +#include "mgmtMnode.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" -#include "mgmtDClient.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtVgroup.h" void *tsChildTableSdb; int32_t tsChildTableUpdateSize; -void *(*mgmtChildTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); - -void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static void mgmtDestroyChildTable(SChildTableObj *pTable) { - free(pTable); -} - -static void mgmtChildTableActionInit() { - mgmtChildTableActionFp[SDB_TYPE_INSERT] = mgmtChildTableActionInsert; - mgmtChildTableActionFp[SDB_TYPE_DELETE] = mgmtChildTableActionDelete; - mgmtChildTableActionFp[SDB_TYPE_UPDATE] = mgmtChildTableActionUpdate; - mgmtChildTableActionFp[SDB_TYPE_ENCODE] = mgmtChildTableActionEncode; - mgmtChildTableActionFp[SDB_TYPE_DECODE] = mgmtChildTableActionDecode; - mgmtChildTableActionFp[SDB_TYPE_RESET] = mgmtChildTableActionReset; - mgmtChildTableActionFp[SDB_TYPE_DESTROY] = mgmtChildTableActionDestroy; -} - -void *mgmtChildTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; - memcpy(pTable, str, tsChildTableUpdateSize); - return NULL; + tfree(pTable); } -void *mgmtChildTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *)row; - mgmtDestroyChildTable(pTable); - return NULL; +static int32_t mgmtChildTableActionDestroy(SSdbOperDesc *pOper) { + mgmtDestroyChildTable(pOper->pObj); + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; +static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { mError("ctable:%s, not in vgroup:%d", pTable->tableId, pTable->vgId); - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in db:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return NULL; - } - - if (!sdbMaster) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid != pTable->sid) { - mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); - return NULL; - } + return TSDB_CODE_INVALID_ACCT; } pTable->superTable = mgmtGetSuperTable(pTable->superTableId); @@ -107,83 +75,65 @@ void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ss mgmtAddTableIntoDb(pDb); mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); - } - - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; +static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return NULL; + return TSDB_CODE_INVALID_ACCT; } mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); mgmtRemoveTableFromDb(pDb); mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - mgmtRemoveTableFromSuperTable(pTable->superTable); - if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pDb, pVgroup); - } - - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtChildTableActionReset(row, str, size, NULL); +static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; + assert(pTable != NULL && pOper->rowData != NULL); - memcpy(str, pTable, tsChildTableUpdateSize); - *ssize = tsChildTableUpdateSize; + memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); + pOper->rowSize = tsChildTableUpdateSize; - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); - - SChildTableObj *pTable = (SChildTableObj *)calloc(sizeof(SChildTableObj), 1); - if (pTable == NULL) return NULL; +static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - if (size < tsChildTableUpdateSize) { - mgmtDestroyChildTable(pTable); - return NULL; + pOper->pObj = calloc(1, sizeof(SChildTableObj)); + if (pOper->pObj == NULL) { + return TSDB_CODE_SERV_OUT_OF_MEMORY; } - memcpy(pTable, str, tsChildTableUpdateSize); - return (void *)pTable; -} + memcpy(pOper->pObj, pOper->rowData, tsChildTableUpdateSize); -void *mgmtChildTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtChildTableActionFp[(uint8_t)action] != NULL) { - return (*(mgmtChildTableActionFp[(uint8_t)action]))(row, str, size, ssize); - } - return NULL; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitChildTables() { @@ -191,12 +141,23 @@ int32_t mgmtInitChildTables() { void *pLastNode = NULL; SChildTableObj *pTable = NULL; - mgmtChildTableActionInit(); SChildTableObj tObj; - tsChildTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "ctables", + .hashSessions = tsMaxTables, + .maxRowSize = tsChildTableUpdateSize, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtChildTableActionInsert, + .deleteFp = mgmtChildTableActionDelete, + .updateFp = mgmtChildTableActionUpdate, + .encodeFp = mgmtChildTableActionEncode, + .decodeFp = mgmtChildTableActionDecode, + .destroyFp = mgmtChildTableActionDestroy, + }; - tsChildTableSdb = sdbOpenTable(tsMaxTables, tsChildTableUpdateSize, - "ctables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtChildTableAction); + tsChildTableSdb = sdbOpenTable(&tableDesc); if (tsChildTableSdb == NULL) { mError("failed to init child table data"); return -1; @@ -212,7 +173,11 @@ int32_t mgmtInitChildTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ctable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -221,7 +186,11 @@ int32_t mgmtInitChildTables() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -230,7 +199,11 @@ int32_t mgmtInitChildTables() { mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -238,28 +211,27 @@ int32_t mgmtInitChildTables() { if (pVgroup->tableList == NULL) { mError("ctable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } - pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1); - SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTable->superTableId); if (pSuperTable == NULL) { mError("ctable:%s, stable:%s not exist", pTable->tableId, pTable->superTableId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } - - pTable->superTable = pSuperTable; - mgmtAddTableIntoSuperTable(pSuperTable); - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); } mTrace("child table is initialized"); @@ -271,8 +243,13 @@ void mgmtCleanUpChildTables() { } void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { - char *pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; - int32_t tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; + char * pTagData = NULL; + int32_t tagDataLen = 0; + if (pMsg != NULL) { + pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; + tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; + } + int32_t totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags; int32_t contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen; @@ -305,18 +282,14 @@ void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTab pSchema++; } - memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); + if (pMsg != NULL) { + memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); + } + return pCreate; } void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { - int32_t numOfTables = sdbGetNumOfRows(tsChildTableSdb); - if (numOfTables >= tsMaxTables) { - mError("ctable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, tsMaxTables); - terrno = TSDB_CODE_TOO_MANY_TABLES; - return NULL; - } - char *pTagData = (char *) pCreate->schema; // it is a tag key SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData); if (pSuperTable == NULL) { @@ -337,12 +310,18 @@ void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->type = TSDB_CHILD_TABLE; pTable->createdTime = taosGetTimestampMs(); pTable->uid = (((uint64_t) pTable->vgId) << 40) + ((((uint64_t) pTable->sid) & ((1ul << 24) - 1ul)) << 16) + - ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sid = tid; pTable->vgId = pVgroup->vgId; pTable->superTable = pSuperTable; - if (sdbInsertRow(tsChildTableSdb, pTable, 0) < 0) { + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbInsertRow(&desc); + + if (sdbInsertRow(&desc) < 0) { free(pTable); mError("ctable:%s, update sdb error", pCreate->tableId); terrno = TSDB_CODE_SDB_ERROR; @@ -485,12 +464,45 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsChildTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); + pNode = pLastNode; + numOfTables++; + continue; + } + } + + mTrace("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); +} + +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { + void *pNode = NULL; + void *pLastNode = NULL; + int32_t numOfTables = 0; + SChildTableObj *pTable = NULL; + + while (1) { + pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); + if (pTable == NULL) { + break; + } + + if (pTable->superTable == pStable) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsChildTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables ++; + numOfTables++; continue; } } - mTrace("db:%s, all child tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("stable:%s, all child tables:%d is dropped from sdb", pStable->tableId, numOfTables); } \ No newline at end of file diff --git a/src/mnode/src/mgmtDClient.c b/src/mnode/src/mgmtDClient.c index da11ad20811734d158aeaa12733ddae178402273..cbdcb8ff5dec6fae0684f8257a5379058622618e 100644 --- a/src/mnode/src/mgmtDClient.c +++ b/src/mnode/src/mgmtDClient.c @@ -42,7 +42,7 @@ int32_t mgmtInitDClient() { rpcInit.label = "MND-DC"; rpcInit.numOfThreads = 1; rpcInit.cfp = mgmtProcessRspFromDnode; - rpcInit.sessions = tsMaxDnodes * 5; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = "mgmtDClient"; diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index 177a45764ccd773a3ceb549fd0dccb624c4257c3..b1d01b91f1f147b600fc70bd696be8e986c3eadf 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -45,7 +45,7 @@ int32_t mgmtInitDServer() { rpcInit.label = "MND-DS"; rpcInit.numOfThreads = 1; rpcInit.cfp = mgmtProcessMsgFromDnode; - rpcInit.sessions = tsMaxDnodes * 5; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = mgmtDServerRetrieveAuth; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 88934599dc237b6451d8631b36db5583c37e819e..0cc793f2c077218ca97cf3dad5230c456022dc52 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -15,33 +15,33 @@ #define _DEFAULT_SOURCE #include "os.h" - -#include "mgmtDb.h" +#include "taoserror.h" +#include "tstatus.h" +#include "tutil.h" +#include "name.h" +#include "mnode.h" #include "mgmtAcct.h" #include "mgmtBalance.h" #include "mgmtChildTable.h" +#include "mgmtDb.h" #include "mgmtDnode.h" #include "mgmtGrant.h" +#include "mgmtShell.h" #include "mgmtMnode.h" #include "mgmtNormalTable.h" -#include "mgmtShell.h" +#include "mgmtChildTable.h" +#include "mgmtSdb.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" -#include "mnode.h" - -#include "taoserror.h" -#include "tstatus.h" -#include "tutil.h" -#include "name.h" static void *tsDbSdb = NULL; static int32_t tsDbUpdateSize; static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); static void mgmtDropDb(void *handle, void *tmrId); -static void mgmtSetDbDirty(SDbObj *pDb); +static int32_t mgmtSetDbDirty(SDbObj *pDb); static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -49,65 +49,93 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg); static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg); -static void *(*mgmtDbActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - -static void mgmtDbActionInit() { - mgmtDbActionFp[SDB_TYPE_INSERT] = mgmtDbActionInsert; - mgmtDbActionFp[SDB_TYPE_DELETE] = mgmtDbActionDelete; - mgmtDbActionFp[SDB_TYPE_UPDATE] = mgmtDbActionUpdate; - mgmtDbActionFp[SDB_TYPE_ENCODE] = mgmtDbActionEncode; - mgmtDbActionFp[SDB_TYPE_DECODE] = mgmtDbActionDecode; - mgmtDbActionFp[SDB_TYPE_RESET] = mgmtDbActionReset; - mgmtDbActionFp[SDB_TYPE_DESTROY] = mgmtDbActionDestroy; +static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) { + tfree(pOper->pObj); + return TSDB_CODE_SUCCESS; } -static void *mgmtDbAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtDbActionFp[(uint8_t)action] != NULL) { - return (*(mgmtDbActionFp[(uint8_t)action]))(row, str, size, ssize); +static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + + pDb->pHead = NULL; + pDb->pTail = NULL; + pDb->prev = NULL; + pDb->next = NULL; + pDb->numOfVgroups = 0; + pDb->numOfTables = 0; + pDb->numOfSuperTables = 0; + + if (pAcct != NULL) { + mgmtAddDbIntoAcct(pAcct, pDb); + } + else { + mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); + return TSDB_CODE_INVALID_ACCT; } - return NULL; + + return TSDB_CODE_SUCCESS; } -int32_t mgmtInitDbs() { - void * pNode = NULL; - SDbObj * pDb = NULL; - SAcctObj *pAcct = NULL; +static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtDbActionInit(); + mgmtRemoveDbFromAcct(pAcct, pDb); + mgmtDropAllNormalTables(pDb); + mgmtDropAllChildTables(pDb); + mgmtDropAllSuperTables(pDb); + mgmtDropAllVgroups(pDb); + + return TSDB_CODE_SUCCESS; +} - SDbObj tObj; - tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; +static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; +} - tsDbSdb = sdbOpenTable(tsMaxDbs, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction); - if (tsDbSdb == NULL) { - mError("failed to init db data"); +static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + + if (pOper->maxRowSize < tsDbUpdateSize) { return -1; + } else { + memcpy(pOper->rowData, pDb, tsDbUpdateSize); + pOper->rowSize = tsDbUpdateSize; + return TSDB_CODE_SUCCESS; } +} - while (1) { - pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb); - if (pDb == NULL) break; +static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { + SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj)); + if (pDb == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; + + memcpy(pDb, pOper->rowData, tsDbUpdateSize); + pOper->pObj = pDb; + return TSDB_CODE_SUCCESS; +} - pDb->pHead = NULL; - pDb->pTail = NULL; - pDb->prev = NULL; - pDb->next = NULL; - pDb->numOfTables = 0; - pDb->numOfVgroups = 0; - pDb->numOfSuperTables = 0; - pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct != NULL) - mgmtAddDbIntoAcct(pAcct, pDb); - else { - mError("db:%s acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); - } +int32_t mgmtInitDbs() { + SDbObj tObj; + tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "dbs", + .hashSessions = TSDB_MAX_DBS, + .maxRowSize = tsDbUpdateSize, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtDbActionInsert, + .deleteFp = mgmtDbActionDelete, + .updateFp = mgmtDbActionUpdate, + .encodeFp = mgmtDbActionEncode, + .decodeFp = mgmtDbActionDecode, + .destroyFp = mgmtDbActionDestroy, + }; + + tsDbSdb = sdbOpenTable(&tableDesc); + if (tsDbSdb == NULL) { + mError("failed to init db data"); + return -1; } mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_DB, mgmtProcessCreateDbMsg); @@ -210,14 +238,14 @@ static int32_t mgmtCheckDBParams(SCMCreateDbMsg *pCreate) { static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { // assign default parameters - if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; // - if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; // - if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; // - if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; // - if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; // - if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; // - if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; // - if (pCreate->compression < 0) pCreate->compression = tsCompression; // + if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; + if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; + if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; + if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; + if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; + if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; + if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; + if (pCreate->compression < 0) pCreate->compression = tsCompression; if (pCreate->commitLog < 0) pCreate->commitLog = tsCommitLog; if (pCreate->replications < 0) pCreate->replications = tsReplications; // if (pCreate->rowsInFileBlock < 0) pCreate->rowsInFileBlock = tsRowsInFileBlock; // @@ -248,18 +276,10 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { pCreate->blocksPerTable = TSDB_MIN_AVG_BLOCKS; } - pCreate->maxSessions++; - return TSDB_CODE_SUCCESS; } static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { - int32_t numOfDbs = sdbGetNumOfRows(tsDbSdb); - if (numOfDbs >= tsMaxDbs) { - mWarn("numOfDbs:%d, exceed tsMaxDbs:%d", numOfDbs, tsMaxDbs); - return TSDB_CODE_TOO_MANY_DATABASES; - } - int32_t code = mgmtCheckDbLimit(pAcct); if (code != 0) { return code; @@ -287,9 +307,17 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { pDb->createdTime = taosGetTimestampMs(); pDb->cfg = *pCreate; - if (sdbInsertRow(tsDbSdb, pDb, 0) < 0) { - code = TSDB_CODE_SDB_ERROR; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj) + }; + + code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { tfree(pDb); + code = TSDB_CODE_SDB_ERROR; } return code; @@ -303,73 +331,8 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) { return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb)); } -static int32_t mgmtAlterDb(SAcctObj *pAcct, SCMAlterDbMsg *pAlter) { - return 0; -// int32_t code = TSDB_CODE_SUCCESS; -// -// SDbObj *pDb = (SDbObj *) sdbGetRow(tsDbSdb, pAlter->db); -// if (pDb == NULL) { -// mTrace("db:%s is not exist", pAlter->db); -// return TSDB_CODE_INVALID_DB; -// } -// -// int32_t oldReplicaNum = pDb->cfg.replications; -// if (pAlter->daysToKeep > 0) { -// mTrace("db:%s daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, pAlter->daysToKeep); -// pDb->cfg.daysToKeep = pAlter->daysToKeep; -// } else if (pAlter->replications > 0) { -// mTrace("db:%s replica:%d change to %d", pDb->name, pDb->cfg.replications, pAlter->replications); -// if (pAlter->replications < TSDB_REPLICA_MIN_NUM || pAlter->replications > TSDB_REPLICA_MAX_NUM) { -// mError("invalid db option replica: %d valid range: %d--%d", pAlter->replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); -// return TSDB_CODE_INVALID_OPTION; -// } -// pDb->cfg.replications = pAlter->replications; -// } else if (pAlter->maxSessions > 0) { -// mTrace("db:%s tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, pAlter->maxSessions); -// if (pAlter->maxSessions < TSDB_MIN_TABLES_PER_VNODE || pAlter->maxSessions > TSDB_MAX_TABLES_PER_VNODE) { -// mError("invalid db option tables: %d valid range: %d--%d", pAlter->maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); -// return TSDB_CODE_INVALID_OPTION; -// } -// if (pAlter->maxSessions < pDb->cfg.maxSessions) { -// mError("invalid db option tables: %d should larger than original:%d", pAlter->maxSessions, pDb->cfg.maxSessions); -// return TSDB_CODE_INVALID_OPTION; -// } -// return TSDB_CODE_INVALID_OPTION; -// //The modification of tables needs to rewrite the head file, so disable this option -// //pDb->cfg.maxSessions = pAlter->maxSessions; -// } else { -// mError("db:%s alter msg, replica:%d, keep:%d, tables:%d, origin replica:%d keep:%d", pDb->name, -// pAlter->replications, pAlter->maxSessions, pAlter->daysToKeep, -// pDb->cfg.replications, pDb->cfg.daysToKeep); -// return TSDB_CODE_INVALID_OPTION; -// } -// -// if (sdbUpdateRow(tsDbSdb, pDb, tsDbUpdateSize, 1) < 0) { -// return TSDB_CODE_SDB_ERROR; -// } -// -// SVgObj *pVgroup = pDb->pHead; -// while (pVgroup != NULL) { -// balanceUpdateVgroupState(pVgroup, TSDB_VG_LB_STATUS_UPDATE, 0); -// if (oldReplicaNum < pDb->cfg.replications) { -// if (!balanceAddVnode(pVgroup, NULL, NULL)) { -// mWarn("db:%s vgroup:%d not enough dnode to add vnode", pAlter->db, pVgroup->vgId); -// code = TSDB_CODE_NO_ENOUGH_DNODES; -// } -// } -// if (pAlter->maxSessions > 0) { -// //rebuild meterList in mgmtVgroup.c -// mgmtUpdateVgroup(pVgroup); -// } -//// mgmtSendCreateVnodeMsg(pVgroup); -// pVgroup = pVgroup->next; -// } -// mgmtStartBalanceTimer(10); -// -// return code; -} - -int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtAddVgroupIntoDb(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; pVgroup->next = pDb->pHead; pVgroup->prev = NULL; @@ -378,11 +341,10 @@ int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { pDb->pHead = pVgroup; pDb->numOfVgroups++; - - return 0; } -int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; pVgroup->next = NULL; pVgroup->prev = pDb->pTail; @@ -391,32 +353,25 @@ int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) { pDb->pTail = pVgroup; pDb->numOfVgroups++; - - return 0; } -int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; if (pVgroup->prev) pVgroup->prev->next = pVgroup->next; if (pVgroup->next) pVgroup->next->prev = pVgroup->prev; if (pVgroup->prev == NULL) pDb->pHead = pVgroup->next; if (pVgroup->next == NULL) pDb->pTail = pVgroup->prev; pDb->numOfVgroups--; - - return 0; } -int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtAddVgroupIntoDbTail(pDb, pVgroup); - - return 0; +void mgmtMoveVgroupToTail(SVgObj *pVgroup) { + mgmtRemoveVgroupFromDb(pVgroup); + mgmtAddVgroupIntoDbTail(pVgroup); } -int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtAddVgroupIntoDb(pDb, pVgroup); - - return 0; +void mgmtMoveVgroupToHead(SVgObj *pVgroup) { + mgmtRemoveVgroupFromDb(pVgroup); + mgmtAddVgroupIntoDb(pVgroup); } void mgmtCleanUpDbs() { @@ -438,7 +393,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -566,7 +521,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) return 0; } -char *mgmtGetDbStr(char *src) { +static char *mgmtGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); return ++pos; @@ -636,7 +591,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * if (strcmp(pUser->user, "root") == 0) { #endif pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxSessions - 1; // table num can be created should minus 1 + *(int32_t *)pWrite = pDb->cfg.maxSessions; // table num can be created should minus 1 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -649,9 +604,9 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; #ifdef _TD_ARM_32_ - *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); + *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); #else - *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); + *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); #endif cols++; @@ -691,68 +646,6 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * return numOfRows; } -void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - - pDb->pHead = NULL; - pDb->pTail = NULL; - pDb->numOfVgroups = 0; - pDb->numOfTables = 0; - mgmtAddDbIntoAcct(pAcct, pDb); - - return NULL; -} - -void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtRemoveDbFromAcct(pAcct, pDb); - - mgmtDropAllNormalTables(pDb); - mgmtDropAllChildTables(pDb); - mgmtDropAllSuperTables(pDb); - - return NULL; -} - -void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtDbActionReset(row, str, size, ssize); -} - -void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - if (size < tsDbUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pDb, tsDbUpdateSize); - *ssize = tsDbUpdateSize; - } - - return NULL; -} -void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) malloc(sizeof(SDbObj)); - if (pDb == NULL) return NULL; - memset(pDb, 0, sizeof(SDbObj)); - - memcpy(pDb, str, tsDbUpdateSize); - - return (void *)pDb; -} - -void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - memcpy(pDb, str, tsDbUpdateSize); - - return NULL; -} - -void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - tfree(row); - return NULL; -} - void mgmtAddSuperTableIntoDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfSuperTables, 1); } @@ -768,8 +661,23 @@ void mgmtRemoveTableFromDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfTables, -1); } -static void mgmtSetDbDirty(SDbObj *pDb) { +static int32_t mgmtSetDbDirty(SDbObj *pDb) { + if (pDb->dirty) return TSDB_CODE_SUCCESS; + pDb->dirty = true; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = tsDbUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_SDB_ERROR; + } + + return code; } static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { @@ -794,32 +702,121 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { } else { code = mgmtCreateDb(pMsg->pUser->pAcct, pCreate); if (code == TSDB_CODE_SUCCESS) { - mLPrint("DB:%s is created by %s", pCreate->db, pMsg->pUser->user); + mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user); } } mgmtSendSimpleResp(pMsg->thandle, code); } +static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { + SDbCfg newCfg = pDb->cfg; + int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t maxSessions = htonl(pAlter->maxSessions); + int8_t replications = pAlter->replications; + + terrno = TSDB_CODE_SUCCESS; + + if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) { + mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep); + newCfg.daysToKeep = daysToKeep; + } else if (replications > 0 && replications != pDb->cfg.replications) { + mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); + if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) { + mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + terrno = TSDB_CODE_INVALID_OPTION; + } + newCfg.replications = replications; + } else if (maxSessions > 0 && maxSessions != pDb->cfg.maxSessions) { + mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, maxSessions); + if (maxSessions < TSDB_MIN_TABLES_PER_VNODE || maxSessions > TSDB_MAX_TABLES_PER_VNODE) { + mError("invalid db option tables: %d valid range: %d--%d", maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + terrno = TSDB_CODE_INVALID_OPTION; + } + if (maxSessions < pDb->cfg.maxSessions) { + mError("invalid db option tables: %d should larger than original:%d", maxSessions, pDb->cfg.maxSessions); + terrno = TSDB_CODE_INVALID_OPTION; + } + newCfg.maxSessions = maxSessions; + } else { + } + + return newCfg; +} + +static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { + SDbCfg newCfg = mgmtGetAlterDbOption(pDb, pAlter); + if (terrno != TSDB_CODE_SUCCESS) { + return terrno; + } + + if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { + pDb->cfg = newCfg; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = tsDbUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_SDB_ERROR; + } + } + + return TSDB_CODE_SUCCESS; +} + static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) { if (mgmtCheckRedirect(pMsg->thandle)) return; SCMAlterDbMsg *pAlter = pMsg->pCont; - pAlter->daysPerFile = htonl(pAlter->daysPerFile); - pAlter->daysToKeep = htonl(pAlter->daysToKeep); - pAlter->maxSessions = htonl(pAlter->maxSessions) + 1; + mTrace("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->thandle); + + if (mgmtCheckExpired()) { + mError("db:%s, failed to alter, grant expired", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); + return; + } - int32_t code; if (!pMsg->pUser->writeAuth) { - code = TSDB_CODE_NO_RIGHTS; - } else { - code = mgmtAlterDb(pMsg->pUser->pAcct, pAlter); - if (code == TSDB_CODE_SUCCESS) { - mLPrint("DB:%s is altered by %s", pAlter->db, pMsg->pUser->user); - } + mError("db:%s, failed to alter, no rights", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); + return; } - mgmtSendSimpleResp(pMsg->thandle, code); + SDbObj *pDb = mgmtGetDb(pAlter->db); + if (pDb == NULL) { + mError("db:%s, failed to alter, invalid db", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB); + return; + } + + int32_t code = mgmtAlterDb(pDb, pAlter); + if (code != TSDB_CODE_SUCCESS) { + mError("db:%s, failed to alter, invalid db option", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, code); + } + + SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); + memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); + pMsg->pCont = NULL; + + SVgObj *pVgroup = pDb->pHead; + if (pVgroup != NULL) { + mPrint("vgroup:%d, will be altered", pVgroup->vgId); + newMsg->ahandle = pVgroup; + newMsg->expected = pVgroup->numOfVnodes; + mgmtAlterVgroup(pVgroup, newMsg); + return; + } + + mTrace("db:%s, all vgroups is altered", pDb->name); + + mgmtSendSimpleResp(newMsg->thandle, TSDB_CODE_SUCCESS); + rpcFreeCont(newMsg->pCont); + free(newMsg); } static void mgmtDropDb(void *handle, void *tmrId) { @@ -827,7 +824,12 @@ static void mgmtDropDb(void *handle, void *tmrId) { SDbObj *pDb = newMsg->ahandle; mPrint("db:%s, drop db from sdb", pDb->name); - int32_t code = sdbDeleteRow(tsDbSdb, pDb); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb + }; + int32_t code = sdbDeleteRow(&oper); if (code != 0) { code = TSDB_CODE_SDB_ERROR; } @@ -874,7 +876,12 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) { return; } - mgmtSetDbDirty(pDb); + int32_t code = mgmtSetDbDirty(pDb); + if (code != TSDB_CODE_SUCCESS) { + mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code)); + mgmtSendSimpleResp(pMsg->thandle, code); + return; + } SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index cf57b43918fb5fd89665d6aee770007a0768a1d5..0aab0834718a2c4215ff6cdc10499e4f35edc4c9 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -547,9 +547,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return ; } - uint32_t lastPrivateIp = pDnode->privateIp; - uint32_t lastPublicIp = pDnode->publicIp; - pDnode->privateIp = htonl(pStatus->privateIp); pDnode->publicIp = htonl(pStatus->publicIp); pDnode->lastReboot = htonl(pStatus->lastReboot); @@ -566,11 +563,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { mgmtSetDnodeMaxVnodes(pDnode); } - if (lastPrivateIp != pDnode->privateIp || lastPublicIp != pDnode->publicIp) { - mgmtUpdateVgroupIp(pDnode); - //mgmtUpdateMnodeIp(); - } - int32_t openVnodes = htons(pStatus->openVnodes); for (int32_t j = 0; j < openVnodes; ++j) { pDnode->vload[j].vgId = htonl(pStatus->load[j].vgId); @@ -599,7 +591,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return; } - mgmtGetMnodeIpList(&pRsp->ipList); + mgmtGetMnodePrivateIpList(&pRsp->ipList); pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus); diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index 66200e5a1407aa59e1e2a76f736c23eb0f82a7b3..ab0b99fa586512e64d1550b29ab77e1b4cbe4332 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -25,6 +25,8 @@ #include "mgmtDClient.h" #include "mgmtDnode.h" #include "mgmtDServer.h" +#include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtVgroup.h" #include "mgmtUser.h" #include "mgmtTable.h" @@ -65,7 +67,7 @@ int32_t mgmtStartSystem() { return 0; } - tsMgmtTmr = taosTmrInit((tsMaxDnodes + tsMaxShellConns) * 3, 200, 3600000, "MND"); + tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND"); if (tsMgmtTmr == NULL) { mError("failed to init timer"); return -1; @@ -109,8 +111,8 @@ int32_t mgmtStartSystem() { return -1; } - if (sdbInitPeers(tsMnodeDir) < 0) { - mError("failed to init peers"); + if (mgmtInitMnodes() < 0) { + mError("failed to init mnodes"); return -1; } @@ -125,7 +127,7 @@ int32_t mgmtStartSystem() { void mgmtStopSystem() { - if (sdbMaster) { + if (mgmtIsMaster()) { mTrace("it is a master mgmt node, it could not be stopped"); return; } @@ -136,7 +138,7 @@ void mgmtStopSystem() { void mgmtCleanUpSystem() { mPrint("starting to clean up mgmt"); - sdbCleanUpPeers(); + mgmtCleanupMnodes(); mgmtCleanupBalance(); mgmtCleanUpShell(); mgmtCleanupDClient(); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index f7e6e3f8cb4c984917a30ff7000e7d4373619c3b..3dfce5e6afa452558bdd1c522361684b13a147f6 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -17,48 +17,93 @@ #include "os.h" #include "trpc.h" #include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtUser.h" -int32_t (*mgmtAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; -int32_t (*mgmtRemoveMnodeFp)(uint32_t privateIp) = NULL; -int32_t (*mgmtGetMnodesNumFp)() = NULL; -void * (*mgmtGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; +int32_t (*mpeerAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; +int32_t (*mpeerRemoveMnodeFp)(uint32_t privateIp) = NULL; +int32_t (*mpeerGetMnodesNumFp)() = NULL; +void * (*mpeerGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; +int32_t (*mpeerInitMnodesFp)() = NULL; +void (*mpeerCleanUpMnodesFp)() = NULL; static SMnodeObj tsMnodeObj = {0}; +static bool tsMnodeIsMaster = false; +static bool tsMnodeIsServing = false; static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static char *mgmtMnodeStatusStr[] = { + "offline", + "unsynced", + "syncing", + "serving", + "null" +}; + +static char *mgmtMnodeRoleStr[] = { + "unauthed", + "undecided", + "master", + "slave", + "null" +}; + +int32_t mgmtInitMnodes() { + if (mpeerInitMnodesFp) { + return (*mpeerInitMnodesFp)(); + } else { + tsMnodeIsServing = true; + tsMnodeIsMaster = true; + return 0; + } +} + +void mgmtCleanupMnodes() { + if (mpeerCleanUpMnodesFp) { + (*mpeerCleanUpMnodesFp)(); + } +} + +bool mgmtInServerStatus() { + return tsMnodeIsServing; +} + +bool mgmtIsMaster() { + return tsMnodeIsMaster; +} + bool mgmtCheckRedirect(void *handle) { return false; } int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp) { - if (mgmtAddMnodeFp) { - return (*mgmtAddMnodeFp)(privateIp, publicIp); + if (mpeerAddMnodeFp) { + return (*mpeerAddMnodeFp)(privateIp, publicIp); } else { return 0; } } int32_t mgmtRemoveMnode(uint32_t privateIp) { - if (mgmtRemoveMnodeFp) { - return (*mgmtRemoveMnodeFp)(privateIp); + if (mpeerRemoveMnodeFp) { + return (*mpeerRemoveMnodeFp)(privateIp); } else { return 0; } } static int32_t mgmtGetMnodesNum() { - if (mgmtGetMnodesNumFp) { - return (*mgmtGetMnodesNumFp)(); + if (mpeerGetMnodesNumFp) { + return (*mpeerGetMnodesNumFp)(); } else { return 1; } } static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) { - if (mgmtGetNextMnodeFp) { - return (*mgmtGetNextMnodeFp)(pShow, pMnode); + if (mpeerGetNextMnodeFp) { + return (*mpeerGetNextMnodeFp)(pShow, pMnode); } else { if (*pMnode == NULL) { *pMnode = &tsMnodeObj; @@ -148,11 +193,11 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbStatusStr[(uint8_t)pMnode->status]); + strcpy(pWrite, mgmtMnodeStatusStr[pMnode->status]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbRoleStr[(uint8_t)pMnode->role]); + strcpy(pWrite, mgmtMnodeRoleStr[pMnode->role]); cols++; tinet_ntoa(ipstr, pMnode->publicIp); @@ -167,7 +212,14 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi return numOfRows; } -void mgmtGetMnodeIpList(SRpcIpSet *ipSet) { +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) { + ipSet->inUse = 0; + ipSet->port = htons(tsMnodeDnodePort); + ipSet->numOfIps = 1; + ipSet->ip[0] = htonl(inet_addr(tsMasterIp)); +} + +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index f076b69f36323f08c0e69195d6d1ab08b4ac4150..4e5e4ce7cc8af2bb99685632dd2b3388a3c247d4 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -26,191 +26,132 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtGrant.h" +#include "mgmtMnode.h" #include "mgmtNormalTable.h" +#include "mgmtSdb.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtVgroup.h" -void *tsNormalTableSdb; +void *tsNormalTableSdb; int32_t tsNormalTableUpdateSize; -void *(*mgmtNormalTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); - -void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { - free(pTable->schema); - free(pTable->sql); - free(pTable); -} - -static void mgmtNormalTableActionInit() { - mgmtNormalTableActionFp[SDB_TYPE_INSERT] = mgmtNormalTableActionInsert; - mgmtNormalTableActionFp[SDB_TYPE_DELETE] = mgmtNormalTableActionDelete; - mgmtNormalTableActionFp[SDB_TYPE_UPDATE] = mgmtNormalTableActionUpdate; - mgmtNormalTableActionFp[SDB_TYPE_ENCODE] = mgmtNormalTableActionEncode; - mgmtNormalTableActionFp[SDB_TYPE_DECODE] = mgmtNormalTableActionDecode; - mgmtNormalTableActionFp[SDB_TYPE_RESET] = mgmtNormalTableActionReset; - mgmtNormalTableActionFp[SDB_TYPE_DESTROY] = mgmtNormalTableActionDestroy; -} - -void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - memcpy(pTable, str, tsNormalTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns) + pTable->sqlLen; - pTable->schema = realloc(pTable->schema, schemaSize); - pTable->sql = (char*)pTable->schema + sizeof(SSchema) * (pTable->numOfColumns); - memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); - - return NULL; + tfree(pTable->schema); + tfree(pTable->sql); + tfree(pTable); } -void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *)row; - mgmtDestroyNormalTable(pTable); - return NULL; +static int32_t mgmtNormalTableActionDestroy(SSdbOperDesc *pOper) { + mgmtDestroyNormalTable(pOper->pObj); + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; +static int32_t mgmtNormalTableActionInsert(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("id:%s not in vgroup:%d", pTable->tableId, pTable->vgId); - return NULL; + mError("ntable:%s not in vgroup:%d", pTable->tableId, pTable->vgId); + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); - return NULL; + mError("ntable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { - mError("account not exists"); - return NULL; - } - - if (!sdbMaster) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid != pTable->sid) { - mError("sid:%d is not matched from the master:%d", sid, pTable->sid); - return NULL; - } + mError("ntable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); + return TSDB_CODE_INVALID_ACCT; } mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); mgmtAddTableIntoDb(pDb); mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); - } - - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; +static int32_t mgmtNormalTableActionDelete(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); - return NULL; + mError("ntable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("account not exists"); - return NULL; + return TSDB_CODE_INVALID_ACCT; } mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1); mgmtRemoveTableFromDb(pDb); mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pDb, pVgroup); - } - - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtNormalTableActionReset(row, str, size, NULL); +static int32_t mgmtNormalTableActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtNormalTableActionEncode(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; + assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (size < tsNormalTableUpdateSize + schemaSize + 1) { - *ssize = -1; - return NULL; + if (pOper->maxRowSize < tsNormalTableUpdateSize + schemaSize) { + return TSDB_CODE_INVALID_MSG_LEN; } - memcpy(str, pTable, tsNormalTableUpdateSize); - memcpy(str + tsNormalTableUpdateSize, pTable->schema, schemaSize); - memcpy(str + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - *ssize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; + memcpy(pOper->rowData, pTable, tsNormalTableUpdateSize); + memcpy(pOper->rowData + tsNormalTableUpdateSize, pTable->schema, schemaSize); + memcpy(pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - return NULL; + pOper->rowSize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static int32_t mgmtNormalTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - SNormalTableObj *pTable = (SNormalTableObj *)malloc(sizeof(SNormalTableObj)); - if (pTable == NULL) { - return NULL; - } - memset(pTable, 0, sizeof(SNormalTableObj)); + SNormalTableObj *pTable = (SNormalTableObj *)calloc(1, sizeof(SNormalTableObj)); + if (pTable == NULL) TSDB_CODE_SERV_OUT_OF_MEMORY; - if (size < tsNormalTableUpdateSize) { - mgmtDestroyNormalTable(pTable); - return NULL; - } - memcpy(pTable, str, tsNormalTableUpdateSize); + memcpy(pTable, pOper->rowData, tsNormalTableUpdateSize); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = (SSchema *)malloc(schemaSize); if (pTable->schema == NULL) { mgmtDestroyNormalTable(pTable); - return NULL; + return -1; } - memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); + memcpy(pTable->schema, pOper->rowData + tsNormalTableUpdateSize, schemaSize); pTable->sql = (char *)malloc(pTable->sqlLen); if (pTable->sql == NULL) { mgmtDestroyNormalTable(pTable); - return NULL; - } - memcpy(pTable->sql, str + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); - return (void *)pTable; -} - -void *mgmtNormalTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtNormalTableActionFp[(uint8_t)action] != NULL) { - return (*(mgmtNormalTableActionFp[(uint8_t)action]))(row, str, size, ssize); + return -1; } - return NULL; + memcpy(pTable->sql, pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); + + pOper->pObj = pTable; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitNormalTables() { @@ -218,12 +159,23 @@ int32_t mgmtInitNormalTables() { void *pLastNode = NULL; SNormalTableObj *pTable = NULL; - mgmtNormalTableActionInit(); SNormalTableObj tObj; - tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsNormalTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "ntables", + .hashSessions = TSDB_MAX_NORMAL_TABLES, + .maxRowSize = sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtNormalTableActionInsert, + .deleteFp = mgmtNormalTableActionDelete, + .updateFp = mgmtNormalTableActionUpdate, + .encodeFp = mgmtNormalTableActionEncode, + .decodeFp = mgmtNormalTableActionDecode, + .destroyFp = mgmtNormalTableActionDestroy, + }; - tsNormalTableSdb = sdbOpenTable(tsMaxTables, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, - "ntables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtNormalTableAction); + tsNormalTableSdb = sdbOpenTable(&tableDesc); if (tsNormalTableSdb == NULL) { mError("failed to init ntables data"); return -1; @@ -237,7 +189,11 @@ int32_t mgmtInitNormalTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ntable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsNormalTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -246,7 +202,11 @@ int32_t mgmtInitNormalTables() { if (pVgroup == NULL) { mError("ntable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -255,7 +215,12 @@ int32_t mgmtInitNormalTables() { mError("ntable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -263,19 +228,14 @@ int32_t mgmtInitNormalTables() { if (pVgroup->tableList == NULL) { mError("ntable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } - - mgmtAddTableIntoVgroup(pVgroup, (STableInfo *)pTable); - //pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1); - - pTable->sql = (char *)pTable->schema + sizeof(SSchema) * pTable->numOfColumns; - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); } mTrace("ntables is initialized"); @@ -323,13 +283,6 @@ void *mgmtBuildCreateNormalTableMsg(SNormalTableObj *pTable) { } void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid) { - int32_t numOfTables = sdbGetNumOfRows(tsNormalTableSdb); - if (numOfTables >= TSDB_MAX_NORMAL_TABLES) { - mError("table:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_NORMAL_TABLES); - terrno = TSDB_CODE_TOO_MANY_TABLES; - return NULL; - } - SNormalTableObj *pTable = (SNormalTableObj *) calloc(sizeof(SNormalTableObj), 1); if (pTable == NULL) { mError("table:%s, failed to alloc memory", pCreate->tableId); @@ -341,7 +294,7 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->type = TSDB_NORMAL_TABLE; pTable->vgId = pVgroup->vgId; pTable->createdTime = taosGetTimestampMs(); - pTable->uid = (((uint64_t) pTable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sid = sid; pTable->sversion = 0; pTable->numOfColumns = htons(pCreate->numOfColumns); @@ -377,7 +330,11 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t mTrace("table:%s, stream sql len:%d sql:%s", pTable->tableId, pTable->sqlLen, pTable->sql); } - if (sdbInsertRow(tsNormalTableSdb, pTable, 0) < 0) { + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + if (sdbInsertRow(&desc) < 0) { mError("table:%s, update sdb error", pTable->tableId); free(pTable); terrno = TSDB_CODE_SDB_ERROR; @@ -474,7 +431,14 @@ int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int3 pTable->sversion++; pAcct->acctInfo.numOfTimeSeries += ncols; - sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + desc.rowData = pTable; + desc.rowSize = tsNormalTableUpdateSize; + sdbUpdateRow(&desc); + return TSDB_CODE_SUCCESS; } @@ -503,7 +467,14 @@ int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) pTable->sversion++; pAcct->acctInfo.numOfTimeSeries--; - sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1); + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + desc.rowData = pTable; + desc.rowSize = tsNormalTableUpdateSize; + sdbUpdateRow(&desc); return TSDB_CODE_SUCCESS; } @@ -564,12 +535,17 @@ void mgmtDropAllNormalTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsNormalTableSdb, pTable); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsNormalTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables ++; + numOfTables++; continue; } } - mTrace("db:%s, all normal tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("db:%s, all normal tables:%d is dropped from sdb", pDropDb->name, numOfTables); } diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index b20e66dd874b6b589087b37e66f463e5d32aee89..1b98ac95966a8c832eabe0beff7fcbd519d97bea 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -762,3 +762,13 @@ int32_t mgmtInitProfile() { void mgmtCleanUpProfile() { } + +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) { + if (pMsg != NULL) { + if (pMsg->pCont != NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + free(pMsg); + } +} \ No newline at end of file diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c new file mode 100644 index 0000000000000000000000000000000000000000..0b1ac3be9706ea8c7342914a4fa87e06ca7cb94c --- /dev/null +++ b/src/mnode/src/mgmtSdb.c @@ -0,0 +1,767 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tchecksum.h" +#include "tglobalcfg.h" +#include "tlog.h" +#include "trpc.h" +#include "tutil.h" +#include "hashint.h" +#include "hashstr.h" +#include "mgmtSdb.h" + +#define abs(x) (((x) < 0) ? -(x) : (x)) +#define SDB_MAX_PEERS 4 +#define SDB_DELIMITER 0xFFF00F00 +#define SDB_ENDCOMMIT 0xAFFFAAAF + +typedef struct { + uint64_t swVersion; + int16_t sdbFileVersion; + char reserved[2]; + TSCKSUM checkSum; +} SSdbHeader; + +typedef struct _SSdbTable { + SSdbHeader header; + char tableName[TSDB_DB_NAME_LEN]; + char fileName[TSDB_FILENAME_LEN]; + ESdbKeyType keyType; + int32_t tableId; + int32_t hashSessions; + int32_t maxRowSize; + int32_t autoIndex; + int32_t fd; + int64_t numOfRows; + int64_t version; + int64_t fileSize; + void * iHandle; + int32_t (*insertFp)(SSdbOperDesc *pDesc); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*destroyFp)(SSdbOperDesc *pOper); + pthread_mutex_t mutex; +} SSdbTable; + +typedef struct { + int64_t version; + int64_t offset; + int32_t rowSize; + void * row; +} SRowMeta; + +typedef struct { + int32_t delimiter; + int32_t rowSize; + int64_t version; + char data[]; +} SRowHead; + +typedef enum { + SDB_FORWARD_TYPE_INSERT, + SDB_FORWARD_TYPE_DELETE, + SDB_FORWARD_TYPE_UPDATE +} ESdbForwardType; + +typedef struct { + ESdbForwardType type; + int32_t tableId; + int64_t version; + int32_t rowSize; + void * rowData; +} SForwardMsg; + +extern char version[]; +const int16_t sdbFileVersion = 2; +int32_t (*mpeerForwardRequestFp)(SForwardMsg *forwardMsg) = NULL; + +static SSdbTable *sdbTableList[10] = {0}; +static int32_t sdbNumOfTables = 0; +static uint64_t sdbVersion = 0; + +static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash}; +static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash}; +static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash}; +static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData}; +static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; +static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData}; + +uint64_t sdbGetVersion() { return sdbVersion; } +int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; } +int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } + +static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { + static char str[16]; + switch (pTable->keyType) { + case SDB_KEY_TYPE_STRING: + return (char *)row; + case SDB_KEY_TYPE_AUTO: + sprintf(str, "%d", *(int32_t *)row); + return str; + default: + return "unknown"; + } +} + +static int32_t sdbForwardDbReqToPeer(SForwardMsg *forwardMsg) { + if (mpeerForwardRequestFp) { + return mpeerForwardRequestFp(forwardMsg); + } else { + return 0; + } +} + +static void sdbFinishCommit(SSdbTable *pTable) { + uint32_t sdbEcommit = SDB_ENDCOMMIT; + off_t offset = lseek(pTable->fd, 0, SEEK_END); + assert(offset == pTable->fileSize); + twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); + pTable->fileSize += sizeof(sdbEcommit); +} + +static int32_t sdbOpenSdbFile(SSdbTable *pTable) { + struct stat fstat, ofstat; + uint64_t size; + char * dirc = NULL; + char * basec = NULL; + union { + char cversion[64]; + uint64_t iversion; + } swVersion; + + memcpy(swVersion.cversion, version, sizeof(uint64_t)); + + // check sdb.db and .sdb.db status + char fn[TSDB_FILENAME_LEN] = "\0"; + dirc = strdup(pTable->fileName); + basec = strdup(pTable->fileName); + sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); + tfree(dirc); + tfree(basec); + if (stat(fn, &ofstat) == 0) { // .sdb.db file exists + if (stat(pTable->fileName, &fstat) == 0) { + remove(fn); + } else { + remove(pTable->fileName); + rename(fn, pTable->fileName); + } + } + + pTable->fd = open(pTable->fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + if (pTable->fd < 0) { + sdbError("table:%s, failed to open file:%s", pTable->tableName, pTable->fileName); + return -1; + } + + pTable->fileSize = 0; + stat(pTable->fileName, &fstat); + size = sizeof(pTable->header); + + if (fstat.st_size == 0) { + pTable->header.swVersion = swVersion.iversion; + pTable->header.sdbFileVersion = sdbFileVersion; + if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) { + sdbError("table:%s, failed to get file header checksum, file:%s", pTable->tableName, pTable->fileName); + tclose(pTable->fd); + return -1; + } + twrite(pTable->fd, &(pTable->header), size); + pTable->fileSize += size; + sdbFinishCommit(pTable); + } else { + uint32_t sdbEcommit = 0; + off_t offset = lseek(pTable->fd, -(sizeof(sdbEcommit)), SEEK_END); + while (offset > 0) { + read(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); + if (sdbEcommit == SDB_ENDCOMMIT) { + ftruncate(pTable->fd, offset + sizeof(sdbEcommit)); + break; + } + offset = lseek(pTable->fd, -(sizeof(sdbEcommit) + 1), SEEK_CUR); + } + lseek(pTable->fd, 0, SEEK_SET); + + ssize_t tsize = read(pTable->fd, &(pTable->header), size); + if (tsize < size) { + sdbError("table:%s, failed to read sdb file header, file:%s", pTable->tableName, pTable->fileName); + tclose(pTable->fd); + return -1; + } + + if (pTable->header.swVersion != swVersion.iversion) { + sdbWarn("table:%s, sdb file:%s version not match software version", pTable->tableName, pTable->fileName); + } + + if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) { + sdbError("table:%s, sdb file header is broken since checksum mismatch, file:%s", pTable->tableName, pTable->fileName); + tclose(pTable->fd); + return -1; + } + + pTable->fileSize += size; + // skip end commit symbol + lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR); + pTable->fileSize += sizeof(sdbEcommit); + } + + pTable->numOfRows = 0; + + return pTable->fd; +} + +static int32_t sdbInitTableByFile(SSdbTable *pTable) { + sdbTrace("table:%s, open sdb file:%s for read", pTable->tableName, pTable->fileName); + if (sdbOpenSdbFile(pTable) < 0) { + sdbError("table:%s, failed to open sdb file:%s for read", pTable->tableName, pTable->fileName); + return -1; + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)malloc(total_size); + if (rowHead == NULL) { + sdbError("table:%s, failed to allocate row head memory, sdb:%s", pTable->tableName, pTable->tableName); + return -1; + } + + int32_t numOfChanged = 0; + int32_t maxAutoIndex = 0; + while (1) { + memset(rowHead, 0, total_size); + + int32_t bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); + if (bytes < 0) { + sdbError("table:%s, failed to read sdb file:%s", pTable->tableName, pTable->fileName); + tfree(rowHead); + return -1; + } + + if (bytes == 0) break; + + if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { + pTable->fileSize++; + lseek(pTable->fd, -(bytes - 1), SEEK_CUR); + continue; + } + + if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { + sdbError("table:%s, error row size in sdb filesize:%d, version:%d rowSize:%d maxRowSize:%d", pTable->tableName, + pTable->fileSize, rowHead->version, rowHead->rowSize, pTable->maxRowSize); + pTable->fileSize += sizeof(SRowHead); + continue; + } + + bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); + if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { + // TODO: Here may cause pTable->fileSize not end of the file + sdbError("table:%s, failed to read sdb file, version:%d rowSize:%d", pTable->tableName, rowHead->version, + rowHead->rowSize); + break; + } + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { + sdbError("table:%s, error sdb checksum, version:%d, skip", pTable->tableName, rowHead->version); + pTable->fileSize += real_size; + continue; + } + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); + } + + pTable->version = MAX(pTable->version, abs(rowHead->version)); + + void *pMetaRow = sdbGetRow(pTable, rowHead->data); + if (pMetaRow == NULL) { + if (rowHead->version < 0) { + sdbError("table:%s, error sdb negative version:%d, record:%s, skip", pTable->tableName, rowHead->version, + sdbGetkeyStr(pTable, rowHead->data)); + } else { + SRowMeta rowMeta; + rowMeta.version = rowHead->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = rowHead->rowSize; + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .rowSize = rowHead->rowSize + }; + int32_t code = (*pTable->decodeFp)(&oper); + if (code == TSDB_CODE_SUCCESS) { + rowMeta.row = oper.pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); + pTable->numOfRows++; + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read new record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } else { + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } + } + } else { + if (rowHead->version < 0) { + SSdbOperDesc oper = { + .table = pTable, + .pObj = pMetaRow + }; + (*pTable->destroyFp)(&oper); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); + pTable->numOfRows--; + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read deleted record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } else { + SRowMeta rowMeta; + rowMeta.version = rowHead->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = rowHead->rowSize; + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .rowSize = rowHead->rowSize, + .pObj = pMetaRow + }; + + (*pTable->destroyFp)(&oper); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); + + int32_t code = (*pTable->decodeFp)(&oper); + if (code == TSDB_CODE_SUCCESS) { + rowMeta.row = oper.pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read updated record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } else { + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); + } + } + numOfChanged++; + } + + pTable->fileSize += real_size; + pTable->fileSize += 4; + lseek(pTable->fd, 4, SEEK_CUR); + } + + void *pNode = NULL; + while (1) { + SRowMeta * pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; + + SSdbOperDesc oper = { + .pObj = pMeta->row, + .table = pTable, + .version = pMeta->version, + }; + + int32_t code = (*pTable->insertFp)(&oper); + if (code != TSDB_CODE_SUCCESS) { + sdbError("table:%s, failed to insert record:%s", pTable->tableName, sdbGetkeyStr(pTable, rowHead->data)); + } + } + + sdbVersion += pTable->version; + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + pTable->autoIndex = maxAutoIndex; + } + + tfree(rowHead); + return 0; +} + +void *sdbOpenTable(SSdbTableDesc *pDesc) { + SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); + if (pTable == NULL) return NULL; + + pTable->keyType = pDesc->keyType; + pTable->hashSessions = pDesc->hashSessions; + pTable->maxRowSize = pDesc->maxRowSize; + pTable->insertFp = pDesc->insertFp; + pTable->deleteFp = pDesc->deleteFp; + pTable->updateFp = pDesc->updateFp; + pTable->encodeFp = pDesc->encodeFp; + pTable->decodeFp = pDesc->decodeFp; + pTable->destroyFp = pDesc->destroyFp; + strcpy(pTable->tableName, pDesc->tableName); + sprintf(pTable->fileName, "%s/%s.db", tsMnodeDir, pTable->tableName); + + if (sdbInitIndexFp[pTable->keyType] != NULL) { + pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); + } + + pthread_mutex_init(&pTable->mutex, NULL); + + if (sdbInitTableByFile(pTable) < 0) return NULL; + + pTable->tableId = sdbNumOfTables++; + sdbTableList[pTable->tableId] = pTable; + + sdbTrace("table:%s, is initialized, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d numOfTables:%d", + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbNumOfTables); + + return pTable; +} + +static SRowMeta *sdbGetRowMeta(void *handle, void *key) { + SSdbTable *pTable = (SSdbTable *)handle; + SRowMeta * pMeta; + + if (handle == NULL) return NULL; + + pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); + + return pMeta; +} + +void *sdbGetRow(void *handle, void *key) { + SSdbTable *pTable = (SSdbTable *)handle; + SRowMeta * pMeta; + + if (handle == NULL) return NULL; + + pthread_mutex_lock(&pTable->mutex); + pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); + pthread_mutex_unlock(&pTable->mutex); + + if (pMeta == NULL) { + return NULL; + } + + return pMeta->row; +} + +int32_t sdbInsertRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) { + sdbError("sdb tables is null"); + return TSDB_CODE_OTHERS; + } + + if (sdbGetRow(pTable, pOper->pObj)) { + sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj)); + return TSDB_CODE_ALREADY_THERE; + } + + pOper->maxRowSize = pTable->maxRowSize; + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_INSERT, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to forward record:%s version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); + pthread_mutex_unlock(&pTable->mutex); + return TSDB_CODE_OTHERS; + } + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + pthread_mutex_unlock(&pTable->mutex); + sdbError("table:%s, failed to allocate row head memory for record:%s version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); + return -1; + } + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; + } + pTable->version++; + sdbVersion++; + + pOper->rowData = rowHead->data; + (*pTable->encodeFp)(pOper); + rowHead->rowSize = pOper->rowSize; + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; + assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("table:%s, failed to get checksum while inserting", pTable->tableName); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + tfree(rowHead); + + // update in SDB layer + SRowMeta rowMeta; + rowMeta.version = pTable->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = pOper->rowSize; + rowMeta.row = pOper->pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); + + pTable->numOfRows++; + + pthread_mutex_unlock(&pTable->mutex); + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, insert record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + + (*pTable->insertFp)(pOper); + + return 0; +} + +// row here can be object or null-terminated string +int32_t sdbDeleteRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) return -1; + + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); + if (pMeta == NULL) { + sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); + return -1; + } + + void * pMetaRow = pMeta->row; + assert(pMetaRow != NULL); + + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_DELETE, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pMeta->rowSize, + .rowData = pMeta->row, + }; + + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to delete record", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + } + + int32_t total_size = sizeof(SRowHead) + pMeta->rowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + + pTable->version++; + sdbVersion++; + + int32_t rowSize = 0; + switch (pTable->keyType) { + case SDB_KEY_TYPE_STRING: + rowSize = strlen((char *)pOper->pObj) + 1; + break; + case SDB_KEY_TYPE_AUTO: + rowSize = sizeof(uint64_t); + break; + default: + return -1; + } + + rowHead->delimiter = SDB_DELIMITER; + rowHead->rowSize = rowSize; + rowHead->version = -(pTable->version); + memcpy(rowHead->data, pOper->pObj, rowSize); + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + + tfree(rowHead); + + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, delete record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + + // Delete from current layer + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); + + pTable->numOfRows--; + + pthread_mutex_unlock(&pTable->mutex); + + (*pTable->deleteFp)(pOper); + (*pTable->destroyFp)(pOper); + + return 0; +} + +// row here can be the object or the string info (encoded string) +int32_t sdbUpdateRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) return -1; + + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); + if (pMeta == NULL) { + sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " version:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version); + return -1; + } + + void *pMetaRow = pMeta->row; + assert(pMetaRow != NULL); + + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_UPDATE, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to update record", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + sdbError("table:%s, failed to allocate row head memory", pTable->tableName); + return -1; + } + + if (pMetaRow != pOper->pObj) { + memcpy(rowHead->data, pOper->rowData, pOper->rowSize); + rowHead->rowSize = pOper->rowSize; + } else { + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .maxRowSize = pTable->maxRowSize, + .pObj = pOper->pObj + }; + (*pTable->encodeFp)(&oper); + rowHead->rowSize = oper.rowSize; + } + + pTable->version++; + sdbVersion++; + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("table:%s, failed to get checksum, version:%d", pTable->tableName, rowHead->version); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, update record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + + pMeta->version = pTable->version; + pMeta->offset = pTable->fileSize; + pMeta->rowSize = rowHead->rowSize; + + pthread_mutex_unlock(&pTable->mutex); + + (*pTable->updateFp)(pOper); // update in upper layer + + tfree(rowHead); + + return 0; +} + +void sdbCloseTable(void *handle) { + SSdbTable *pTable = (SSdbTable *)handle; + void * pNode = NULL; + + if (pTable == NULL) return; + + while (1) { + SRowMeta * pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; + + SSdbOperDesc oper = { + .pObj = pMeta->row, + .table = pTable, + .version = pMeta->version, + }; + + (*pTable->destroyFp)(&oper); + } + + if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); + + if (pTable->fd) tclose(pTable->fd); + + pthread_mutex_destroy(&pTable->mutex); + + sdbNumOfTables--; + sdbTrace("table:%s, is closed, version:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); + + tfree(pTable); +} + +void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { + SSdbTable *pTable = (SSdbTable *)handle; + SRowMeta * pMeta; + + *ppRow = NULL; + if (pTable == NULL) return NULL; + + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) return NULL; + + *ppRow = pMeta->row; + + return pNode; +} diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 9cf0e3b57338f113996579d9cb129013691fdea9..3ff78b985e4cb55d07d3284f750a56bc4fb99faf 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -32,6 +32,7 @@ #include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" @@ -63,7 +64,7 @@ int32_t mgmtInitShell() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_HEARTBEAT, mgmtProcessHeartBeatMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg); - tsMgmtTranQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, 1, "mnodeT"); + tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT"); int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0; if (numOfThreads < 1) { @@ -131,7 +132,7 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) { } static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { - if (sdbGetRunStatus() != SDB_STATUS_SERVING) { + if (!mgmtInServerStatus()) { mgmtProcessMsgWhileNotReady(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; @@ -309,20 +310,10 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { return; } - pHBRsp->ipList.inUse = 0; - pHBRsp->ipList.port = htons(tsMnodeShellPort); - pHBRsp->ipList.numOfIps = 0; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - pHBRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps); - if (connInfo.serverIp == tsPublicIpInt) { - for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) { - pHBRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]); - } - } else { - for (int i = 0; i < pSdbIpList->numOfIps; ++i) { - pHBRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]); - } - } + if (connInfo.serverIp == tsPublicIpInt) { + mgmtGetMnodePublicIpList(&pHBRsp->ipList); + } else { + mgmtGetMnodePrivateIpList(&pHBRsp->ipList); } /* @@ -411,20 +402,11 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { strcpy(pConnectRsp->serverVersion, version); pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->superAuth = pUser->superAuth; - pConnectRsp->ipList.inUse = 0; - pConnectRsp->ipList.port = htons(tsMnodeShellPort); - pConnectRsp->ipList.numOfIps = 0; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - pConnectRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps); - if (connInfo.serverIp == tsPublicIpInt) { - for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) { - pConnectRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]); - } - } else { - for (int i = 0; i < pSdbIpList->numOfIps; ++i) { - pConnectRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]); - } - } + + if (connInfo.serverIp == tsPublicIpInt) { + mgmtGetMnodePublicIpList(&pConnectRsp->ipList); + } else { + mgmtGetMnodePrivateIpList(&pConnectRsp->ipList); } connect_over: diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index bd697ade93288917e38bb1388049d98717bdd9c3..f2b54ec000091914f49a5f1b94b97cffc1bed8e6 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -23,6 +23,7 @@ #include "mgmtGrant.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" +#include "mgmtSdb.h" #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" @@ -31,160 +32,106 @@ #include "name.h" #include "tsqlfunction.h" -static void *tsSuperTableSdb; +static void *tsSuperTableSdb; static int32_t tsSuperTableUpdateSize; -static void *(*mgmtSuperTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static void mgmtDestroySuperTable(SSuperTableObj *pTable) { - free(pTable->schema); - free(pTable); +static void mgmtDestroySuperTable(SSuperTableObj *pStable) { + tfree(pStable->schema); + tfree(pStable); } -static void mgmtSuperTableActionInit() { - SSuperTableObj tObj; - tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - mgmtSuperTableActionFp[SDB_TYPE_INSERT] = mgmtSuperTableActionInsert; - mgmtSuperTableActionFp[SDB_TYPE_DELETE] = mgmtSuperTableActionDelete; - mgmtSuperTableActionFp[SDB_TYPE_UPDATE] = mgmtSuperTableActionUpdate; - mgmtSuperTableActionFp[SDB_TYPE_ENCODE] = mgmtSuperTableActionEncode; - mgmtSuperTableActionFp[SDB_TYPE_DECODE] = mgmtSuperTableActionDecode; - mgmtSuperTableActionFp[SDB_TYPE_RESET] = mgmtSuperTableActionReset; - mgmtSuperTableActionFp[SDB_TYPE_DESTROY] = mgmtSuperTableActionDestroy; -} - -void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - memcpy(pTable, str, tsSuperTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = realloc(pTable->schema, schemaSize); - memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize); - - return NULL; -} - -void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - mgmtDestroySuperTable(pTable); - return NULL; +static int32_t mgmtSuperTableActionDestroy(SSdbOperDesc *pOper) { + mgmtDestroySuperTable(pOper->pObj); + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - STableInfo *pTable = (STableInfo *) row; - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb) { +static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) { + STableInfo *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + if (pDb != NULL) { mgmtAddSuperTableIntoDb(pDb); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - STableInfo *pTable = (STableInfo *) row; - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb) { +static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) { + STableInfo *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + if (pDb != NULL) { mgmtRemoveSuperTableFromDb(pDb); + mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtSuperTableActionReset(row, str, size, NULL); +static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) { + SSuperTableObj *pStable = pOper->pObj; + assert(pOper->pObj != NULL && pOper->rowData != NULL); - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - if (size < tsSuperTableUpdateSize + schemaSize + 1) { - *ssize = -1; - return NULL; + if (pOper->maxRowSize < tsSuperTableUpdateSize + schemaSize) { + return TSDB_CODE_INVALID_MSG_LEN; } - memcpy(str, pTable, tsSuperTableUpdateSize); - memcpy(str + tsSuperTableUpdateSize, pTable->schema, schemaSize); - *ssize = tsSuperTableUpdateSize + schemaSize; + memcpy(pOper->rowData, pStable, tsSuperTableUpdateSize); + memcpy(pOper->rowData + tsSuperTableUpdateSize, pStable->schema, schemaSize); + pOper->rowSize = tsSuperTableUpdateSize + schemaSize; - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - SSuperTableObj *pTable = (SSuperTableObj *) malloc(sizeof(SSuperTableObj)); - if (pTable == NULL) { - return NULL; - } - memset(pTable, 0, sizeof(SSuperTableObj)); + SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj)); + if (pStable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - if (size < tsSuperTableUpdateSize) { - mgmtDestroySuperTable(pTable); - return NULL; - } - memcpy(pTable, str, tsSuperTableUpdateSize); + memcpy(pStable, pOper->rowData, tsSuperTableUpdateSize); - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = malloc(schemaSize); - if (pTable->schema == NULL) { - mgmtDestroySuperTable(pTable); - return NULL; + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); + pStable->schema = malloc(schemaSize); + if (pStable->schema == NULL) { + mgmtDestroySuperTable(pStable); + return -1; } - memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize); - return (void *) pTable; -} + memcpy(pStable->schema, pOper->rowData + tsSuperTableUpdateSize, schemaSize); + pOper->pObj = pStable; -void *mgmtSuperTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtSuperTableActionFp[(uint8_t) action] != NULL) { - return (*(mgmtSuperTableActionFp[(uint8_t) action]))(row, str, size, ssize); - } - return NULL; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitSuperTables() { - void *pNode = NULL; - void *pLastNode = NULL; - SSuperTableObj *pTable = NULL; - - mgmtSuperTableActionInit(); - - tsSuperTableSdb = sdbOpenTable(tsMaxTables, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, - "stables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtSuperTableAction); + SSuperTableObj tObj; + tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "stables", + .hashSessions = TSDB_MAX_SUPER_TABLES, + .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtSuperTableActionInsert, + .deleteFp = mgmtSuperTableActionDelete, + .updateFp = mgmtSuperTableActionUpdate, + .encodeFp = mgmtSuperTableActionEncode, + .decodeFp = mgmtSuperTableActionDecode, + .destroyFp = mgmtSuperTableActionDestroy, + }; + + tsSuperTableSdb = sdbOpenTable(&tableDesc); if (tsSuperTableSdb == NULL) { mError("failed to init stables data"); return -1; } - pNode = NULL; - while (1) { - pNode = sdbFetchRow(tsSuperTableSdb, pNode, (void **) &pTable); - if (pTable == NULL) { - break; - } - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("super table:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsSuperTableSdb, pTable); - pNode = pLastNode; - continue; - } - - mgmtAddSuperTableIntoDb(pDb); - } - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mgmtGetShowSuperTableMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mgmtRetrieveShowSuperTables); @@ -197,13 +144,7 @@ void mgmtCleanUpSuperTables() { } int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { - int32_t numOfTables = sdbGetNumOfRows(tsSuperTableSdb); - if (numOfTables >= TSDB_MAX_SUPER_TABLES) { - mError("stable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_SUPER_TABLES); - return TSDB_CODE_TOO_MANY_TABLES; - } - - SSuperTableObj *pStable = (SSuperTableObj *)calloc(sizeof(SSuperTableObj), 1); + SSuperTableObj *pStable = (SSuperTableObj *)calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; } @@ -213,7 +154,7 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { pStable->createdTime = taosGetTimestampMs(); pStable->vgId = 0; pStable->sid = 0; - pStable->uid = (((uint64_t) pStable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + pStable->uid = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pStable->sversion = 0; pStable->numOfColumns = htons(pCreate->numOfColumns); pStable->numOfTags = htons(pCreate->numOfTags); @@ -235,13 +176,21 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { tschema[col].bytes = htons(tschema[col].bytes); } - if (sdbInsertRow(tsSuperTableSdb, pStable, 0) < 0) { - mError("stable:%s, update sdb error", pStable->tableId); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable, + .rowSize = sizeof(SSuperTableObj) + schemaSize + }; + + int32_t code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mgmtDestroySuperTable(pStable); return TSDB_CODE_SDB_ERROR; + } else { + mLPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); + return TSDB_CODE_SUCCESS; } - - mPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); - return TSDB_CODE_SUCCESS; } int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pStable) { @@ -249,10 +198,14 @@ int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pSta mError("stable:%s, numOfTables:%d not 0", pStable->tableId, pStable->numOfTables); return TSDB_CODE_OTHERS; } else { - //TODO: drop child tables - mError("stable:%s, is dropped from sdb", pStable->tableId); - mgmtRemoveSuperTableFromDb(pDb); - return TSDB_CODE_OTHERS; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable + }; + int32_t code = sdbDeleteRow(&oper); + mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->tableId, tstrerror(code)); + return code; } } @@ -261,10 +214,9 @@ void* mgmtGetSuperTable(char *tableId) { } void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) { - //TODO get vgroup of dnodes SCMSuperTableInfoRsp *rsp = rpcMallocCont(sizeof(SCMSuperTableInfoRsp) + sizeof(uint32_t) * mgmtGetDnodesNum()); - rsp->numOfDnodes = 1; - rsp->dnodeIps[0] = 0; + rsp->numOfDnodes = htonl(1); + rsp->dnodeIps[0] = htonl(inet_addr(tsPrivateIp)); return rsp; } @@ -321,7 +273,7 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ntags * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->tableId); return TSDB_CODE_SUCCESS; @@ -354,7 +306,7 @@ int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -367,8 +319,8 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN return TSDB_CODE_INVALID_MSG_TYPE; } - int32_t rowSize = 0; - uint32_t len = strlen(newTagName); + int32_t rowSize = 0; + uint32_t len = strlen(newTagName); if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { return TSDB_CODE_APP_ERROR; @@ -384,9 +336,10 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN if (msg == NULL) return TSDB_CODE_APP_ERROR; memset(msg, 0, size); - mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); + // mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); - int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, rowSize, 1); + int32_t ret = 0; + // int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); tfree(msg); if (ret < 0) { @@ -448,7 +401,7 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32 pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -481,7 +434,7 @@ int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { pStable->schema = realloc(pStable->schema, schemaSize); pAcct->acctInfo.numOfTimeSeries -= (pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -620,14 +573,19 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsSuperTableSdb, pTable); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsSuperTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; numOfTables ++; continue; } } - mTrace("db:%s, all super tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable) { diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 2fe6e31306ecb49e81f31b6fab8c3d93843ccab5..675de57924ccb46650085510079459a6c678f454 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -22,10 +22,12 @@ #include "mgmtDClient.h" #include "mgmtDb.h" #include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtGrant.h" #include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" #include "mgmtUser.h" @@ -60,6 +62,7 @@ static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg); static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle); +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg); int32_t mgmtInitTables() { int32_t code = mgmtInitSuperTables(); @@ -77,8 +80,6 @@ int32_t mgmtInitTables() { return code; } - mgmtSetVgroupIdPool(); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_TABLE, mgmtProcessCreateTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_TABLE, mgmtProcessDropTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_TABLE, mgmtProcessAlterTableMsg); @@ -91,6 +92,7 @@ int32_t mgmtInitTables() { mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropTableRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropStableRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_TABLE, mgmtProcessTableCfgMsg); return TSDB_CODE_SUCCESS; } @@ -201,13 +203,13 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "table_name"); + strcpy(pSchema[cols].name, "table name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -219,7 +221,7 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "stable_name"); + strcpy(pSchema[cols].name, "stable name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -237,9 +239,6 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) return 0; } -/* - * remove the hole in result set - */ static void mgmtVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { if (rows < capacity) { for (int32_t i = 0; i < numOfCols; ++i) { @@ -433,11 +432,13 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { pTable = mgmtCreateChildTable(pCreate, pVgroup, sid); if (pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } pMDCreate = mgmtBuildCreateChildTableMsg(pCreate, (SChildTableObj *) pTable); if (pMDCreate == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } } else { @@ -445,11 +446,13 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { pTable = mgmtCreateNormalTable(pCreate, pVgroup, sid); if (pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); if (pMDCreate == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } } @@ -760,9 +763,19 @@ static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code != TSDB_CODE_SUCCESS) { if (pTable->type == TSDB_CHILD_TABLE) { - sdbDeleteRow(tsChildTableSdb, pTable); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); } else if (pTable->type == TSDB_NORMAL_TABLE){ - sdbDeleteRow(tsNormalTableSdb, pTable); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsNormalTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); } else {} mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); @@ -816,14 +829,26 @@ static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { } if (pTable->type == TSDB_CHILD_TABLE) { - if (sdbDeleteRow(tsChildTableSdb, pTable) < 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { mError("table:%s, update ctables sdb error", pTable->tableId); mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); free(queueMsg); return; } } else if (pTable->type == TSDB_NORMAL_TABLE){ - if (sdbDeleteRow(tsNormalTableSdb, pTable) < 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsNormalTableSdb, + .pObj = pTable + }; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { mError("table:%s, update ntables sdb error", pTable->tableId); mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); free(queueMsg); @@ -844,32 +869,48 @@ static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) { mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); } -// -// -//static void mgmtProcessTableCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *thandle) { -// SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) pCont; -// pCfg->dnode = htonl(pCfg->dnode); -// pCfg->vnode = htonl(pCfg->vnode); -// pCfg->sid = htonl(pCfg->sid); -// mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// -// if (!sdbMaster) { -// mError("dnode:%s, vnode:%d, sid:%d, not master, redirect it", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0); -// return; -// } -// -// STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); -// if (pTable == NULL) { -// mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_INVALID_TABLE, NULL, 0); -// return; -// } -// -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0); -// -// //TODO -// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); -// mgmtSendCreateTableMsg(NULL, &ipSet, NULL); -//} -// \ No newline at end of file +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont; + pCfg->dnode = htonl(pCfg->dnode); + pCfg->vnode = htonl(pCfg->vnode); + pCfg->sid = htonl(pCfg->sid); + mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + + STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); + if (pTable == NULL) { + mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_TABLE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SMDCreateTableMsg *pMDCreate = NULL; + if (pTable->type == TSDB_CHILD_TABLE) { + mTrace("table:%s, is a child table, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + pMDCreate = mgmtBuildCreateChildTableMsg(NULL, (SChildTableObj *) pTable); + if (pMDCreate == NULL) { + return; + } + } else if (pTable->type == TSDB_NORMAL_TABLE) { + mTrace("table:%s, is a normal table, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); + if (pMDCreate == NULL) { + return; + } + } else { + mError("table:%s, invalid msg type, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + } + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + SRpcMsg rpcRsp = { + .handle = NULL, + .pCont = pMDCreate, + .contLen = htonl(pMDCreate->contLen), + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE + }; + mgmtSendMsgToDnode(&ipSet, &rpcRsp); +} diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 771c78c2229a2191cb64973a81553b026e38926f..576b326669e6bdd8e29e505cdd350a31147ccf6d 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -17,9 +17,11 @@ #include "os.h" #include "trpc.h" #include "ttime.h" +#include "tutil.h" #include "mgmtAcct.h" #include "mgmtGrant.h" #include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" @@ -36,48 +38,84 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg); static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg); -static void *(*mgmtUserActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize); -static void mgmtUserActionInit(); +static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) { + tfree(pOper->pObj); + return TSDB_CODE_SUCCESS; +} -int32_t mgmtInitUsers() { - void *pNode = NULL; - SUserObj *pUser = NULL; - SAcctObj *pAcct = NULL; - int32_t numOfUsers = 0; +static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - mgmtUserActionInit(); + if (pAcct != NULL) { + mgmtAddUserIntoAcct(pAcct, pUser); + } + else { + mError("user:%s, acct:%s info not exist in sdb", pUser->user, pUser->acct); + return TSDB_CODE_INVALID_ACCT; + } - SUserObj tObj; - tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + return TSDB_CODE_SUCCESS; +} - tsUserSdb = sdbOpenTable(tsMaxUsers, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction); - if (tsUserSdb == NULL) { - mError("failed to init user data"); +static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); + + mgmtRemoveUserFromAcct(pAcct, pUser); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) { + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; + + if (pOper->maxRowSize < tsUserUpdateSize) { return -1; + } else { + memcpy(pOper->rowData, pUser, tsUserUpdateSize); + pOper->rowSize = tsUserUpdateSize; + return TSDB_CODE_SUCCESS; } +} - while (1) { - pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser); - if (pUser == NULL) break; - - pUser->prev = NULL; - pUser->next = NULL; +static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { + SUserObj *pUser = (SUserObj *) calloc(1, sizeof(SUserObj)); + if (pUser == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - pAcct = mgmtGetAcct(pUser->acct); - mgmtAddUserIntoAcct(pAcct, pUser); + memcpy(pUser, pOper->rowData, tsUserUpdateSize); + pOper->pObj = pUser; + return TSDB_CODE_SUCCESS; +} - numOfUsers++; +int32_t mgmtInitUsers() { + SUserObj tObj; + tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "users", + .hashSessions = TSDB_MAX_USERS, + .maxRowSize = tsUserUpdateSize, + .keyType = SDB_KEY_TYPE_STRING, + .insertFp = mgmtUserActionInsert, + .deleteFp = mgmtUserActionDelete, + .updateFp = mgmtUserActionUpdate, + .encodeFp = mgmtUserActionEncode, + .decodeFp = mgmtUserActionDecode, + .destroyFp = mgmtUserActionDestroy, + }; + + tsUserSdb = sdbOpenTable(&tableDesc); + if (tsUserSdb == NULL) { + mError("failed to init user data"); + return -1; } - pAcct = mgmtGetAcct("root"); + SAcctObj *pAcct = mgmtGetAcct("root"); mgmtCreateUser(pAcct, "root", "taosdata"); mgmtCreateUser(pAcct, "monitor", tsInternalPass); mgmtCreateUser(pAcct, "_root", tsInternalPass); @@ -101,16 +139,23 @@ SUserObj *mgmtGetUser(char *name) { } static int32_t mgmtUpdateUser(SUserObj *pUser) { - return sdbUpdateRow(tsUserSdb, pUser, 0, 1); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, + .rowSize = tsUserUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pUser); + code = TSDB_CODE_SDB_ERROR; + } + + return code; } static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { - int32_t numOfUsers = sdbGetNumOfRows(tsUserSdb); - if (numOfUsers >= tsMaxUsers) { - mWarn("numOfUsers:%d, exceed tsMaxUsers:%d", numOfUsers, tsMaxUsers); - return TSDB_CODE_TOO_MANY_USERS; - } - int32_t code = mgmtCheckUserLimit(pAcct); if (code != 0) { return code; @@ -131,8 +176,7 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { return code; } - pUser = malloc(sizeof(SUserObj)); - memset(pUser, 0, sizeof(SUserObj)); + pUser = calloc(1, sizeof(SUserObj)); strcpy(pUser->user, name); taosEncryptPass((uint8_t*) pass, strlen(pass), pUser->pass); strcpy(pUser->acct, pAcct->user); @@ -143,8 +187,15 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { pUser->superAuth = 1; } - code = TSDB_CODE_SUCCESS; - if (sdbInsertRow(tsUserSdb, pUser, 0) < 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj) + }; + + code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { tfree(pUser); code = TSDB_CODE_SDB_ERROR; } @@ -165,9 +216,18 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { return TSDB_CODE_NO_RIGHTS; } - sdbDeleteRow(tsUserSdb, pUser); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser + }; - return 0; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_SDB_ERROR; + } + + return code; } static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { @@ -250,83 +310,6 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void return numOfRows; } -static void mgmtUserActionInit() { - mgmtUserActionFp[SDB_TYPE_INSERT] = mgmtUserActionInsert; - mgmtUserActionFp[SDB_TYPE_DELETE] = mgmtUserActionDelete; - mgmtUserActionFp[SDB_TYPE_UPDATE] = mgmtUserActionUpdate; - mgmtUserActionFp[SDB_TYPE_ENCODE] = mgmtUserActionEncode; - mgmtUserActionFp[SDB_TYPE_DECODE] = mgmtUserActionDecode; - mgmtUserActionFp[SDB_TYPE_RESET] = mgmtUserActionReset; - mgmtUserActionFp[SDB_TYPE_DESTROY] = mgmtUserActionDestroy; -} - -static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtUserActionFp[(uint8_t) action] != NULL) { - return (*(mgmtUserActionFp[(uint8_t) action]))(row, str, size, ssize); - } - return NULL; -} - -static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - pUser->pAcct = pAcct; - mgmtAddUserIntoAcct(pAcct, pUser); - - return NULL; -} - -static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - mgmtRemoveUserFromAcct(pAcct, pUser); - - return NULL; -} - -static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtUserActionReset(row, str, size, ssize); -} - -static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - - if (size < tsUserUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pUser, tsUserUpdateSize); - *ssize = tsUserUpdateSize; - } - - return NULL; -} - -static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) malloc(sizeof(SUserObj)); - if (pUser == NULL) return NULL; - memset(pUser, 0, sizeof(SUserObj)); - - memcpy(pUser, str, tsUserUpdateSize); - - return (void *)pUser; -} - -static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *)row; - - memcpy(pUser, str, tsUserUpdateSize); - - return NULL; -} - -static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - tfree(row); - - return NULL; -} - SUserObj *mgmtGetUserFromConn(void *pConn) { SRpcConnInfo connInfo; if (rpcGetConnInfo(pConn, &connInfo) == 0) { @@ -393,7 +376,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { memset(pUser->pass, 0, sizeof(pUser->pass)); taosEncryptPass((uint8_t*)pAlter->pass, strlen(pAlter->pass), pUser->pass); code = mgmtUpdateUser(pUser); - mLPrint("user:%s password is altered by %s, code:%d", pAlter->user, pUser->user, code); + mLPrint("user:%s password is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } else { code = TSDB_CODE_NO_RIGHTS; } @@ -428,10 +411,6 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } if (hasRight) { - //if (pAlter->privilege == 1) { // super - // pUser->superAuth = 1; - // pUser->writeAuth = 1; - //} if (pAlter->privilege == 2) { // read pUser->superAuth = 0; pUser->writeAuth = 0; @@ -442,7 +421,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } code = mgmtUpdateUser(pUser); - mLPrint("user:%s privilege is altered by %s, code:%d", pAlter->user, pUser->user, code); + mLPrint("user:%s privilege is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } else { code = TSDB_CODE_NO_RIGHTS; } @@ -492,7 +471,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { if (hasRight) { code = mgmtDropUser(pUser->pAcct, pDrop->user); if (code == TSDB_CODE_SUCCESS) { - mLPrint("user:%s is dropped by %s", pDrop->user, pUser->user); + mLPrint("user:%s is dropped by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } } else { code = TSDB_CODE_NO_RIGHTS; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index f84b5443a5bd289c78bf66f3672fc4626fe722d9..9e52574b8c8890289f12770229675e99d46cb749 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -23,104 +23,151 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtTable.h" #include "mgmtVgroup.h" -static void *tsVgroupSdb = NULL; +static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; -static void *(*mgmtVgroupActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg); static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg); +static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) ; static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); static void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); -static void mgmtVgroupActionInit() { - SVgObj tObj; - tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - mgmtVgroupActionFp[SDB_TYPE_INSERT] = mgmtVgroupActionInsert; - mgmtVgroupActionFp[SDB_TYPE_DELETE] = mgmtVgroupActionDelete; - mgmtVgroupActionFp[SDB_TYPE_UPDATE] = mgmtVgroupActionUpdate; - mgmtVgroupActionFp[SDB_TYPE_ENCODE] = mgmtVgroupActionEncode; - mgmtVgroupActionFp[SDB_TYPE_DECODE] = mgmtVgroupActionDecode; - mgmtVgroupActionFp[SDB_TYPE_RESET] = mgmtVgroupActionReset; - mgmtVgroupActionFp[SDB_TYPE_DESTROY] = mgmtVgroupActionDestroy; +static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + if (pVgroup->idPool) { + taosIdPoolCleanUp(pVgroup->idPool); + pVgroup->idPool = NULL; + } + if (pVgroup->tableList) { + tfree(pVgroup->tableList); + } + tfree(pOper->pObj); + return TSDB_CODE_SUCCESS; } -static void *mgmtVgroupAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtVgroupActionFp[(uint8_t) action] != NULL) { - return (*(mgmtVgroupActionFp[(uint8_t) action]))(row, str, size, ssize); +static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + if (pDb == NULL) { + return TSDB_CODE_INVALID_DB; } - return NULL; -} -int32_t mgmtInitVgroups() { - void *pNode = NULL; - SVgObj *pVgroup = NULL; + pVgroup->pDb = pDb; + pVgroup->prev = NULL; + pVgroup->next = NULL; - mgmtVgroupActionInit(); + int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; + pVgroup->tableList = (STableInfo **)calloc(pDb->cfg.maxSessions, sizeof(STableInfo *)); + if (pVgroup->tableList == NULL) { + mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size); + return -1; + } - tsVgroupSdb = sdbOpenTable(tsMaxVGroups, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction); - if (tsVgroupSdb == NULL) { - mError("failed to init vgroups data"); + pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); + if (pVgroup->idPool == NULL) { + mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId); + tfree(pVgroup->tableList); return -1; } - while (1) { - pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); - if (pVgroup == NULL) break; + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); + pVgroup->vnodeGid[i].privateIp = pDnode->privateIp; + pVgroup->vnodeGid[i].publicIp = pDnode->publicIp; + pVgroup->vnodeGid[i].vnode = pVgroup->vgId; + } - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb == NULL) continue; + mgmtAddVgroupIntoDb(pVgroup); - pVgroup->prev = NULL; - pVgroup->next = NULL; + return TSDB_CODE_SUCCESS; +} - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)malloc(size); - if (pVgroup->tableList == NULL) { - mError("failed to malloc(size:%d) for the tableList of vgroups", size); - return -1; - } - - memset(pVgroup->tableList, 0, size); +static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + + if (pVgroup->pDb != NULL) { + mgmtRemoveVgroupFromDb(pVgroup); + } - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); - if (pVgroup->idPool == NULL) { - mError("failed to taosInitIdPool for vgroups"); - free(pVgroup->tableList); - return -1; - } - - taosIdPoolReinit(pVgroup->idPool); + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - if (tsIsCluster && pVgroup->vnodeGid[0].publicIp == 0) { - pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp); - pVgroup->vnodeGid[0].privateIp = inet_addr(tsPrivateIp); - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); + SDbObj *pDb = pVgroup->pDb; + if (pDb != NULL) { + if (pDb->cfg.maxSessions != oldTables) { + mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); + taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); + int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; + pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size); } + } + + mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxSessions, pVgroup->numOfVnodes); + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + if (pOper->maxRowSize < tsVgUpdateSize) { + return -1; + } else { + memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); + pOper->rowSize = tsVgUpdateSize; + return TSDB_CODE_SUCCESS; + } +} + +static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { + SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj)); + if (pVgroup == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); + memcpy(pVgroup, pOper->rowData, tsVgUpdateSize); + pOper->pObj = pVgroup; + return TSDB_CODE_SUCCESS; +} + +int32_t mgmtInitVgroups() { + SVgObj tObj; + tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableName = "vgroups", + .hashSessions = TSDB_MAX_VGROUPS, + .maxRowSize = tsVgUpdateSize, + .keyType = SDB_KEY_TYPE_AUTO, + .insertFp = mgmtVgroupActionInsert, + .deleteFp = mgmtVgroupActionDelete, + .updateFp = mgmtVgroupActionUpdate, + .encodeFp = mgmtVgroupActionEncode, + .decodeFp = mgmtVgroupActionDecode, + .destroyFp = mgmtVgroupActionDestroy, + }; + + tsVgroupSdb = sdbOpenTable(&tableDesc); + if (tsVgroupSdb == NULL) { + mError("failed to init vgroups data"); + return -1; } mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mgmtGetVgroupMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mgmtRetrieveVgroups); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mgmtProcessCreateVnodeRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mgmtProcessDropVnodeRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg); mTrace("vgroup is initialized"); return 0; @@ -139,32 +186,41 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg) { if (pDb == NULL) { mError("failed to create vgroup, db not found"); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB); + mgmtFreeQueuedMsg(pMsg); return; } - SVgObj *pVgroup = (SVgObj *)calloc(sizeof(SVgObj), 1); + SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); strcpy(pVgroup->dbName, pDb->name); pVgroup->numOfVnodes = pDb->cfg.replications; + pVgroup->createdTime = taosGetTimestampMs(); if (mgmtAllocVnodes(pVgroup) != 0) { mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes); free(pVgroup); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES); + mgmtFreeQueuedMsg(pMsg); return; } - pVgroup->createdTime = taosGetTimestampMs(); - pVgroup->tableList = (STableInfo **) calloc(sizeof(STableInfo *), pDb->cfg.maxSessions); - pVgroup->numOfTables = 0; - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); - - mgmtAddVgroupIntoDb(pDb, pVgroup); - // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj) + }; - sdbInsertRow(tsVgroupSdb, pVgroup, 0); + int32_t code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pVgroup); + code = TSDB_CODE_SDB_ERROR; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR); + mgmtFreeQueuedMsg(pMsg); + return; + } mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - mPrint("vgroup:%d, dnode:%d vnode:%d", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode); + mPrint("vgroup:%d, index:%d, dnode:%d vnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode); } pMsg->ahandle = pVgroup; @@ -178,28 +234,12 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) { } else { mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mgmtSendDropVgroupMsg(pVgroup, NULL); - sdbDeleteRow(tsVgroupSdb, pVgroup); - } -} - -void mgmtSetVgroupIdPool() { - void * pNode = NULL; - SVgObj *pVgroup = NULL; - SDbObj *pDb; - - while (1) { - pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); - if (pVgroup == NULL || pVgroup->idPool == 0) break; - - taosIdPoolSetFreeList(pVgroup->idPool); - pVgroup->numOfTables = taosIdPoolNumOfUsed(pVgroup->idPool); - - pDb = mgmtGetDb(pVgroup->dbName); - pDb->numOfTables += pVgroup->numOfTables; - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1) - mgmtAddVgroupIntoDbTail(pDb, pVgroup); - else - mgmtAddVgroupIntoDb(pDb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + sdbDeleteRow(&oper); } } @@ -230,7 +270,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vgroup_status"); + strcpy(pSchema[cols].name, "vgroup status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -270,13 +310,13 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vnode_status"); + strcpy(pSchema[cols].name, "vnode status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 16; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "public_ip"); + strcpy(pSchema[cols].name, "public ip"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } @@ -388,109 +428,30 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo return numOfRows; } -static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = row; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - pVgroup->vnodeGid[i].vnode = pVgroup->vgId; - } - - return NULL; -} - -static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = row; - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - - if (pDb != NULL) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - } - - // mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes); - tfree(pVgroup->tableList); - - return NULL; -} - -static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - mgmtVgroupActionReset(row, str, size, ssize); - - SVgObj *pVgroup = (SVgObj *) row; - int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb != NULL) { - if (pDb->cfg.maxSessions != oldTables) { - mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); - taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size); - } - } - - mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); - - return NULL; -} - -static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - if (size < tsVgUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pVgroup, tsVgUpdateSize); - *ssize = tsVgUpdateSize; - } - - return NULL; -} - -static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) malloc(sizeof(SVgObj)); - if (pVgroup == NULL) return NULL; - memset(pVgroup, 0, sizeof(SVgObj)); - - int32_t tsVgUpdateSize = pVgroup->updateEnd - (int8_t *) pVgroup; - memcpy(pVgroup, str, tsVgUpdateSize); - - return (void *) pVgroup; -} - -static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - memcpy(pVgroup, str, tsVgUpdateSize); - return NULL; -} - -static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - if (pVgroup->idPool) { - taosIdPoolCleanUp(pVgroup->idPool); - pVgroup->idPool = NULL; - } - if (pVgroup->tableList) tfree(pVgroup->tableList); - tfree(row); - return NULL; -} - -void mgmtUpdateVgroup(SVgObj *pVgroup) { - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 0); -} - void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { - pVgroup->numOfTables++; - if (pTable->sid >= 0) + if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] == NULL) { pVgroup->tableList[pTable->sid] = pTable; + taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables++; + } + + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + mgmtAddVgroupIntoDbTail(pVgroup); } void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { - pVgroup->numOfTables--; - if (pTable->sid >= 0) + if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] != NULL) { pVgroup->tableList[pTable->sid] = NULL; - taosFreeId(pVgroup->idPool, pTable->sid); + taosFreeId(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables--; + } + + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + mgmtAddVgroupIntoDbTail(pVgroup); } SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + SDbObj *pDb = pVgroup->pDb; if (pDb == NULL) return NULL; SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg)); @@ -514,6 +475,7 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SVnodeDesc *vpeerDesc = pVnode->vpeerDesc; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { + vpeerDesc[j].vnode = htonl(pVgroup->vnodeGid[j].vnode); vpeerDesc[j].ip = htonl(pVgroup->vnodeGid[j].privateIp); } @@ -603,13 +565,23 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { newMsg->contLen = queueMsg->contLen; newMsg->pCont = rpcMallocCont(newMsg->contLen); memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); + queueMsg->pCont = NULL; mgmtAddToShellQueue(newMsg); } else { - sdbDeleteRow(tsVgroupSdb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + int32_t code = sdbDeleteRow(&oper); + if (code != 0) { + code = TSDB_CODE_SDB_ERROR; + } + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); } - free(queueMsg); + mgmtFreeQueuedMsg(queueMsg); } static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) { @@ -659,7 +631,15 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (queueMsg->received != queueMsg->expected) return; - sdbDeleteRow(tsVgroupSdb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + int32_t code = sdbDeleteRow(&oper); + if (code != 0) { + code = TSDB_CODE_SDB_ERROR; + } SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); newMsg->msgType = queueMsg->msgType; @@ -671,50 +651,64 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); mgmtAddToShellQueue(newMsg); - free(queueMsg); + queueMsg->pCont = NULL; + mgmtFreeQueuedMsg(queueMsg); +} + +static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont; + pCfg->dnode = htonl(pCfg->dnode); + pCfg->vnode = htonl(pCfg->vnode); + + SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode); + if (pVgroup == NULL) { + mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL); } -void mgmtUpdateVgroupIp(SDnodeObj *pDnode) { - void * pNode = NULL; +void mgmtDropAllVgroups(SDbObj *pDropDb) { + void *pNode = NULL; + void *pLastNode = NULL; + int32_t numOfTables = 0; + int32_t dbNameLen = strlen(pDropDb->name); SVgObj *pVgroup = NULL; + while (1) { pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); if (pVgroup == NULL) break; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *vnodeGid = pVgroup->vnodeGid + i; - if (vnodeGid->dnodeId == pDnode->dnodeId) { - mPrint("vgroup:%d, dnode:%d, privateIp:%s change to %s, publicIp:%s change to %s", - pVgroup->vgId, vnodeGid->dnodeId, pDnode->privateIp, taosIpStr(vnodeGid->privateIp), - pDnode->publicIp, taosIpStr(vnodeGid->publicIp)); - vnodeGid->publicIp = pDnode->publicIp; - vnodeGid->privateIp = pDnode->privateIp; - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); - } + if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + }; + sdbDeleteRow(&oper); + pNode = pLastNode; + numOfTables++; + continue; } } + + mTrace("db:%s, all vgroups is dropped from sdb", pDropDb->name, numOfTables); } -//static void mgmtProcessVnodeCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *pConn) { -// if (!sdbMaster) { -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0); -// return; -// } -// -// SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) pCont; -// pCfg->dnode = htonl(pCfg->dnode); -// pCfg->vnode = htonl(pCfg->vnode); -// -// SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode); -// if (pVgroup == NULL) { -// mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_NOT_ACTIVE_VNODE, NULL, 0); -// return; -// } -// -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0); -// -// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); -// mgmtSendCreateVnodeMsg(pVgroup, pCfg->vnode, &ipSet, NULL); -//} -// \ No newline at end of file +void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { + assert(ahandle != NULL); + + if (pVgroup->numOfVnodes != pVgroup->pDb->cfg.replications) { + // TODO: + // mgmtSendAlterVgroupMsg(pVgroup, NULL); + } else { + mgmtAddToShellQueue(ahandle); + } +} \ No newline at end of file diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 346170cde8a00049bf05b0cbb9d8fc9ad62d03ce..4ce606f599b8e04f8db50ab30999ebc1d837c274 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -33,7 +33,7 @@ typedef struct SData { } SData; enum { - ST_QUERY_KILLED = 0, // query killed +// ST_QUERY_KILLED = 0, // query killed ST_QUERY_PAUSED = 1, // query paused, due to full of the response buffer ST_QUERY_COMPLETED = 2, // query completed }; @@ -143,7 +143,7 @@ typedef struct SQuery { int32_t pos; int64_t pointsOffset; // the number of points offset to save read data SData** sdata; - int32_t capacity; + int32_t capacity; SSingleColumnFilterInfo* pFilterInfo; } SQuery; @@ -170,21 +170,19 @@ typedef struct SQueryRuntimeEnv { } SQueryRuntimeEnv; typedef struct SQInfo { - uint64_t signature; - void* pVnode; + void* signature; + void* pVnode; TSKEY startTime; - int64_t elapsedTime; + TSKEY elapsedTime; SResultRec rec; - int32_t pointsReturned; int32_t pointsInterpo; int32_t code; // error code to returned to client - int32_t killed; // denotes if current query is killed +// int32_t killed; // denotes if current query is killed sem_t dataReady; SArray* pTableIdList; // table list SQueryRuntimeEnv runtimeEnv; int32_t subgroupIdx; int32_t offset; /* offset in group result set of subgroup */ -// tSidSet* pSidSet; T_REF_DECLARE() /* @@ -226,7 +224,12 @@ void qSuperTableQuery(void* pReadMsg); */ int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo, int32_t *numOfRows, int32_t* rowsize); - -//int32_t qBuildQueryResult(SQInfo* pQInfo, void* pBuf); +/** + * + * @param pQInfo + * @param pRsp + * @return + */ +int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen); #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 04ff9e56f8238ca393ed50a4e975191d158517dd..c7aa57200cff7e7fb6091e55dd4366875f6e57b7 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -219,7 +219,7 @@ typedef struct SQLAggFuncElem { void (*distSecondaryMergeFunc)(SQLFunctionCtx *pCtx); - int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus); + int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); } SQLAggFuncElem; typedef struct SPatternCompareInfo { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 69ae0c06bd936678e5bf75b16b28638274c784c7..cbce5097ec77f538f16d2d24e4f6817a4dd06746 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -64,38 +64,24 @@ typedef struct SPointInterpoSupporter { } SPointInterpoSupporter; typedef enum { - - /* - * the program will call this function again, if this status is set. - * used to transfer from QUERY_RESBUF_FULL - */ + // when query starts to execute, this status will set QUERY_NOT_COMPLETED = 0x1u, - /* - * output buffer is full, so, the next query will be employed, - * in this case, we need to set the appropriated start scan point for - * the next query. - * - * this status is only exist in group-by clause and - * diff/add/division/multiply/ query. + /* result output buffer is full, current query is paused. + * this status is only exist in group-by clause and diff/add/division/multiply/ query. */ QUERY_RESBUF_FULL = 0x2u, - /* - * query is over - * 1. this status is used in one row result query process, e.g., - * count/sum/first/last/ - * avg...etc. - * 2. when the query range on timestamp is satisfied, it is also denoted as - * query_compeleted + /* query is over + * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. + * 2. when all data within queried time window, it is also denoted as query_completed */ QUERY_COMPLETED = 0x4u, - - /* - * all data has been scanned, so current search is stopped, - * At last, the function will transfer this status to QUERY_COMPLETED + + /* when the result is not completed return to client, this status will be + * usually used in case of interval query with interpolation option */ - QUERY_NO_DATA_TO_CHECK = 0x8u, + QUERY_OVER = 0x8u, } vnodeQueryStatus; static void setQueryStatus(SQuery *pQuery, int8_t status); @@ -119,7 +105,7 @@ static int32_t flushFromResultBuf(SQInfo *pQInfo); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); -static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char *primaryColumnData, int32_t size, +static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag); static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); @@ -427,15 +413,9 @@ static bool isTopBottomQuery(SQuery *pQuery) { return false; } -static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - int32_t columnIndex) { - // no SField info exist, or column index larger than the output column, no result. - if (pStatis == NULL) { - return NULL; - } - +static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t index) { // for a tag column, no corresponding field info - SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[columnIndex].pBase.colInfo; + SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[index].pBase.colInfo; if (TSDB_COL_IS_TAG(pColIndexEx->flag)) { return NULL; } @@ -453,13 +433,31 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo return NULL; } +/** + * @param pQuery + * @param col + * @param pDataBlockInfo + * @param pStatis + * @param pColStatis + * @return + */ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SDataStatis **pColStatis) { - if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[col].pBase.colInfo.flag) || pStatis == NULL) { + SColIndexEx* pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; + if (TSDB_COL_IS_TAG(pColIndex->flag)) { return false; } - - *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); + + // query on primary timestamp column, not null value at all + if (pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return false; + } + + *pColStatis = NULL; + if (pStatis != NULL) { + *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); + } + if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { return false; } @@ -887,7 +885,6 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 * the remain meter may not have the required column in cache actually. * So, the validation of required column in cache with the corresponding meter schema is reinforced. */ - if (pDataBlock == NULL) { return NULL; } @@ -939,10 +936,11 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; SDataStatis *tpField = NULL; - bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); - char * dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + + bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); + char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, (char *)primaryKeyCol, pDataBlockInfo->size, functionId, tpField, + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->size, functionId, tpField, hasNull, &sasArray[k], pRuntimeEnv->scanFlag); } @@ -1289,7 +1287,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat if (pRuntimeEnv->pTSBuf != NULL) { // if timestamp filter list is empty, quit current query if (!tsBufNextPos(pRuntimeEnv->pTSBuf)) { - setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + setQueryStatus(pQuery, QUERY_COMPLETED); break; } } @@ -1381,7 +1379,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl return 0; } -void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char *primaryColumnData, int32_t size, +void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag) { pCtx->scanFlag = scanFlag; @@ -1396,15 +1394,15 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char * pCtx->preAggVals.isSet = false; } - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (primaryColumnData != NULL)) { - pCtx->ptsList = (int64_t *)(primaryColumnData); + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (tsCol != NULL)) { + pCtx->ptsList = tsCol; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { // last_dist or first_dist function // store the first&last timestamp into the intermediate buffer [1], the true // value may be null but timestamp will never be null - pCtx->ptsList = (int64_t *)(primaryColumnData); + pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { /* @@ -1420,7 +1418,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char * pTWAInfo->EKey = pQuery->window.ekey; } - pCtx->ptsList = (int64_t *)(primaryColumnData); + pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_ARITHM) { pCtx->param[1].pz = param; @@ -1609,10 +1607,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); } -bool isQueryKilled(SQuery *pQuery) { - return false; - - SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); +static bool isQueryKilled(SQuery *pQuery) { #if 0 /* * check if the queried meter is going to be deleted. @@ -1626,9 +1621,14 @@ bool isQueryKilled(SQuery *pQuery) { return (pQInfo->killed == 1); #endif + return 0; } +static bool setQueryKilled(SQInfo* pQInfo) { + pQInfo->code = TSDB_CODE_QUERY_CANCELLED; +} + bool isFixedOutputQuery(SQuery *pQuery) { if (pQuery->intervalTime != 0) { return false; @@ -2524,22 +2524,18 @@ static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow) { } SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo, SDataStatis **pStatis) { - SQuery * pQuery = pRuntimeEnv->pQuery; - tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; + SQuery *pQuery = pRuntimeEnv->pQuery; uint32_t r = 0; SArray * pDataBlock = NULL; - // STimeWindow *w = &pQueryHandle->window; - if (pQuery->numOfFilterCols > 0) { r = BLK_DATA_ALL_NEEDED; } else { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; int32_t colId = pQuery->pSelectExpr[i].pBase.colInfo.colId; - - // r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], w->skey, w->ekey, colId); + r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId); } if (pRuntimeEnv->pTSBuf > 0 || isIntervalQuery(pQuery)) { @@ -2553,7 +2549,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl // pBlock->keyFirst, pBlock->keyLast, pBlock->numOfPoints); } else if (r == BLK_DATA_FILEDS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { - // return DISK_DATA_LOAD_FAILED; +// return DISK_DATA_LOAD_FAILED; } if (pStatis == NULL) { @@ -2656,7 +2652,6 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (tsdbNextDataBlock(pQueryHandle)) { // check if query is killed or not set the status of query to pass the status check if (isQueryKilled(pQuery)) { - setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return cnt; } @@ -2690,10 +2685,8 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &numOfRes, &pRuntimeEnv->windowResInfo, pDataBlock); - // dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", fileId:%d, slot:%d, pos:%d, rows:%d, - // checked:%d", - // GET_QINFO_ADDR(pQuery), blockInfo.window.skey, blockInfo.window.ekey, pQueryHandle->cur.fileId, - // pQueryHandle->cur.slot, pQuery->pos, blockInfo.size, forwardStep); + dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d", + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.size); // save last access position cnt += forwardStep; @@ -2708,7 +2701,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } if (isIntervalQuery(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { int32_t step = QUERY_IS_ASC_QUERY(pQuery) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP; closeAllTimeWindow(&pRuntimeEnv->windowResInfo); @@ -3036,7 +3029,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { total += pData->numOfElems; } - pQuery->sdata[0]->num = total; + int32_t rows = total; int32_t offset = 0; for (int32_t num = 0; num < list.size; ++num) { @@ -3044,7 +3037,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - char * pDest = pQuery->sdata[i]->data; + char * pDest = pQuery->sdata[i]; memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->numOfElems, bytes * pData->numOfElems); @@ -3055,7 +3048,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { assert(pQuery->rec.pointsRead == 0); - pQuery->rec.pointsRead += pQuery->sdata[0]->num; + pQuery->rec.pointsRead += rows; pQInfo->offset += 1; } @@ -3222,7 +3215,8 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. int32_t pageId = -1; - int32_t remain = pQuery->sdata[0]->num; + assert(0); + int32_t remain = 0;//pQuery->sdata[0]->num; int32_t offset = 0; while (remain > 0) { @@ -3253,12 +3247,12 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; + pCtx[k].aOutputBuf = pQuery->sdata[k] - pCtx[k].outputBytes; pCtx[k].size = 1; pCtx[k].startOffset = 0; pCtx[k].resultInfo = &pResultInfo[k]; - pQuery->sdata[k]->num = 0; + pQuery->sdata[k] = 0; } } @@ -3624,7 +3618,7 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { /* check if query is killed or not */ if (isQueryKilled(pQuery)) { - setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); +// setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } } @@ -4080,11 +4074,6 @@ int32_t vnodeGetResultSize(void *thandle, int32_t *numOfRows) { } } -int64_t vnodeGetOffsetVal(void *thandle) { - SQInfo *pQInfo = (SQInfo *)thandle; - return pQInfo->runtimeEnv.pQuery->limit.offset; -} - bool vnodeHasRemainResults(void *handle) { SQInfo *pQInfo = (SQInfo *)handle; @@ -4109,7 +4098,7 @@ bool vnodeHasRemainResults(void *handle) { } // query has completed - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision); // int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY @@ -4162,62 +4151,13 @@ static int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, tFilePage **p } static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { -#if 0 - SMeterObj *pObj = pQInfo->pObj; - SQuery * pQuery = &pQInfo->query; - - int tnumOfRows = vnodeList[pObj->vnode].cfg.rowsInFileBlock; - - // for metric query, bufIndex always be 0. - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { // pQInfo->bufIndex == 0 + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { int32_t bytes = pQuery->pSelectExpr[col].resBytes; memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); data += bytes * numOfRows; } -#endif -} - -/** - * Copy the result data/file to output message buffer. - * If the result is in file format, read file from disk and copy to output buffer, compression is not involved since - * data in file is already compressed. - * In case of other result in buffer, compress the result before copy once the tsComressMsg is set. - * - * @param handle - * @param data - * @param numOfRows the number of rows that are not returned in current retrieve - * @return - */ -int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows) { - SQInfo *pQInfo = (SQInfo *)handle; - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - assert(pQuery->pSelectExpr != NULL && pQuery->numOfOutputCols > 0); - - // load data from file to msg buffer - if (isTSCompQuery(pQuery)) { - int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); - - // make sure file exist - if (FD_VALID(fd)) { - size_t s = lseek(fd, 0, SEEK_END); - dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); - - lseek(fd, 0, SEEK_SET); - read(fd, data, s); - close(fd); - - unlink(pQuery->sdata[0]->data); - } else { - dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, - pQuery->sdata[0]->data, strerror(errno)); - } - } else { - doCopyQueryResultToMsg(pQInfo, numOfRows, data); - } - - return numOfRows; } int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage **pDataSrc, int32_t numOfRows, @@ -4308,7 +4248,7 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param, void* tsdb) { +int32_t initQInfo(SQInfo *pQInfo, void *param, void* tsdb) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t code = TSDB_CODE_SUCCESS; @@ -4319,7 +4259,7 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param, void* tsdb) { pQuery->window.ekey, pQuery->order.order); sem_post(&pQInfo->dataReady); - pQInfo->killed = 1; + setQueryStatus(pQuery, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; } @@ -4342,7 +4282,6 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param, void* tsdb) { taosArrayPush(cols, &pQuery->colList[i]); } - pQInfo->runtimeEnv.pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; @@ -4993,8 +4932,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { // vnodePrintQueryStatistics(pSupporter); } - dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->rec.pointsRead, - pQInfo->rec.pointsRead, pQInfo->pointsReturned); + dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.pointsRead, pQInfo->rec.pointsTotal); return; } #if 0 @@ -5073,7 +5011,7 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { // since the numOfOutputElems must be identical for all sql functions that are allowed to be executed simutanelously. pQuery->rec.pointsRead = getNumOfResult(pRuntimeEnv); // assert(pQuery->pointsRead <= pQuery->pointsToRead && - // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)); + // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED)); // must be top/bottom query if offset > 0 if (pQuery->limit.offset > 0) { @@ -5177,7 +5115,7 @@ static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= c; } - if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { break; } @@ -5227,7 +5165,7 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.pointsRead, &numOfInterpo); dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.pointsRead); - if (pQuery->rec.pointsRead > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + if (pQuery->rec.pointsRead > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { doRevisedResultsByLimit(pQInfo); break; } @@ -5255,17 +5193,20 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { } void qTableQuery(SQInfo *pQInfo) { - assert(pQInfo != NULL); - - if (pQInfo->killed) { - dTrace("QInfo:%p it is already killed, abort", pQInfo); + if (pQInfo == NULL || pQInfo->signature != pQInfo) { + dTrace("%p freed abort query", pQInfo); return; } - + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - - // dTrace("vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d, QInfo:%p", pQInfo); + + SQuery *pQuery = pRuntimeEnv->pQuery; + if (isQueryKilled(pQuery)) { + dTrace("QInfo:%p it is already killed, abort", pQInfo); + return; + } + + dTrace("QInfo:%p query task is launched", pQInfo); if (vnodeHasRemainResults(pQInfo)) { /* @@ -5291,10 +5232,10 @@ void qTableQuery(SQInfo *pQInfo) { } // here we have scan all qualified data in both data file and cache - if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { // continue to get push data from the group result if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || - (pQuery->intervalTime > 0 && pQInfo->pointsReturned < pQuery->limit.limit)) { + (pQuery->intervalTime > 0 && pQInfo->rec.pointsTotal < pQuery->limit.limit)) { // todo limit the output for interval query? pQuery->rec.pointsRead = 0; pQInfo->subgroupIdx = 0; // always start from 0 @@ -5352,10 +5293,8 @@ void qTableQuery(SQInfo *pQInfo) { /* check if query is killed or not */ if (isQueryKilled(pQuery)) { dTrace("QInfo:%p query is killed", pQInfo); - // pQInfo->over = 1; } else { - // dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", pQInfo, - // pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); + dTrace("QInfo:%p query task completed, %d points are returned", pQInfo, pQuery->rec.pointsRead); } sem_post(&pQInfo->dataReady); @@ -5436,11 +5375,6 @@ static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryTableMsg) { return -1; } - if (pQueryTableMsg->numOfTagsCols < 0 || pQueryTableMsg->numOfTagsCols > TSDB_MAX_TAGS + 1) { - dError("qmsg:%p illegal value of numOfTagsCols %d", pQueryTableMsg, pQueryTableMsg->numOfTagsCols); - return -1; - } - if (pQueryTableMsg->numOfCols <= 0 || pQueryTableMsg->numOfCols > TSDB_MAX_COLUMNS) { dError("qmsg:%p illegal value of numOfCols %d", pQueryTableMsg, pQueryTableMsg->numOfCols); return -1; @@ -5469,34 +5403,66 @@ static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryTableMsg) { return 0; } -static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList) { - pQueryTableMsg->vgId = htons(pQueryTableMsg->vgId); - pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); - - pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); - pQueryTableMsg->window.ekey = htobe64(pQueryTableMsg->window.ekey); +static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArray** pTableIdList) { + assert(pQueryTableMsg->numOfTables > 0); + + *pTableIdList = taosArrayInit(pQueryTableMsg->numOfTables, sizeof(STableIdInfo)); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->sid = htonl(pTableIdInfo->sid); + pTableIdInfo->uid = htobe64(pTableIdInfo->uid); + pTableIdInfo->key = htobe64(pTableIdInfo->key); + + taosArrayPush(*pTableIdList, pTableIdInfo); + pMsg += sizeof(STableIdInfo); + + for (int32_t j = 1; j < pQueryTableMsg->numOfTables; ++j) { + pTableIdInfo = (STableIdInfo *)pMsg; + + pTableIdInfo->sid = htonl(pTableIdInfo->sid); + pTableIdInfo->uid = htobe64(pTableIdInfo->uid); + pTableIdInfo->key = htobe64(pTableIdInfo->key); + + taosArrayPush(*pTableIdList, pTableIdInfo); + pMsg += sizeof(STableIdInfo); + } + + return pMsg; +} - pQueryTableMsg->order = htons(pQueryTableMsg->order); - pQueryTableMsg->orderColId = htons(pQueryTableMsg->orderColId); +/** + * pQueryTableMsg->head has been converted before this function is called. + * + * @param pQueryTableMsg + * @param pTableIdList + * @param pExpr + * @return + */ +static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr) { + pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); - pQueryTableMsg->queryType = htons(pQueryTableMsg->queryType); + pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); + pQueryTableMsg->window.ekey = htobe64(pQueryTableMsg->window.ekey); + pQueryTableMsg->intervalTime = htobe64(pQueryTableMsg->intervalTime); + pQueryTableMsg->slidingTime = htobe64(pQueryTableMsg->slidingTime); + + pQueryTableMsg->limit = htobe64(pQueryTableMsg->limit); + pQueryTableMsg->offset = htobe64(pQueryTableMsg->offset); + + pQueryTableMsg->order = htons(pQueryTableMsg->order); + pQueryTableMsg->orderColId = htons(pQueryTableMsg->orderColId); - pQueryTableMsg->intervalTime = htobe64(pQueryTableMsg->intervalTime); - pQueryTableMsg->slidingTime = htobe64(pQueryTableMsg->slidingTime); + pQueryTableMsg->queryType = htons(pQueryTableMsg->queryType); - pQueryTableMsg->numOfTagsCols = htons(pQueryTableMsg->numOfTagsCols); - pQueryTableMsg->numOfCols = htons(pQueryTableMsg->numOfCols); + pQueryTableMsg->numOfCols = htons(pQueryTableMsg->numOfCols); pQueryTableMsg->numOfOutputCols = htons(pQueryTableMsg->numOfOutputCols); pQueryTableMsg->numOfGroupCols = htons(pQueryTableMsg->numOfGroupCols); - pQueryTableMsg->tagLength = htons(pQueryTableMsg->tagLength); + pQueryTableMsg->tagLength = htons(pQueryTableMsg->tagLength); - pQueryTableMsg->limit = htobe64(pQueryTableMsg->limit); - pQueryTableMsg->offset = htobe64(pQueryTableMsg->offset); - - pQueryTableMsg->tsOffset = htonl(pQueryTableMsg->tsOffset); - pQueryTableMsg->tsLen = htonl(pQueryTableMsg->tsLen); + pQueryTableMsg->tsOffset = htonl(pQueryTableMsg->tsOffset); + pQueryTableMsg->tsLen = htonl(pQueryTableMsg->tsLen); pQueryTableMsg->tsNumOfBlocks = htonl(pQueryTableMsg->tsNumOfBlocks); - pQueryTableMsg->tsOrder = htonl(pQueryTableMsg->tsOrder); + pQueryTableMsg->tsOrder = htonl(pQueryTableMsg->tsOrder); // query msg safety check if (validateQueryMeterMsg(pQueryTableMsg) != 0) { @@ -5506,23 +5472,23 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTa char *pMsg = (char *)(pQueryTableMsg->colList) + sizeof(SColumnInfo) * pQueryTableMsg->numOfCols; for (int32_t col = 0; col < pQueryTableMsg->numOfCols; ++col) { - pQueryTableMsg->colList[col].colId = htons(pQueryTableMsg->colList[col].colId); - pQueryTableMsg->colList[col].type = htons(pQueryTableMsg->colList[col].type); - pQueryTableMsg->colList[col].bytes = htons(pQueryTableMsg->colList[col].bytes); - pQueryTableMsg->colList[col].numOfFilters = htons(pQueryTableMsg->colList[col].numOfFilters); - - assert(pQueryTableMsg->colList[col].type >= TSDB_DATA_TYPE_BOOL && - pQueryTableMsg->colList[col].type <= TSDB_DATA_TYPE_NCHAR); + SColumnInfo* pColInfo = &pQueryTableMsg->colList[col]; + + pColInfo->colId = htons(pColInfo->colId); + pColInfo->type = htons(pColInfo->type); + pColInfo->bytes = htons(pColInfo->bytes); + pColInfo->numOfFilters = htons(pColInfo->numOfFilters); - int32_t numOfFilters = pQueryTableMsg->colList[col].numOfFilters; + assert(pColInfo->type >= TSDB_DATA_TYPE_BOOL && pColInfo->type <= TSDB_DATA_TYPE_NCHAR); + int32_t numOfFilters = pColInfo->numOfFilters; if (numOfFilters > 0) { - pQueryTableMsg->colList[col].filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); + pColInfo->filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); } for (int32_t f = 0; f < numOfFilters; ++f) { SColumnFilterInfo *pFilterInfo = (SColumnFilterInfo *)pMsg; - SColumnFilterInfo *pDestFilterInfo = &pQueryTableMsg->colList[col].filters[f]; + SColumnFilterInfo *pDestFilterInfo = &pColInfo->filters[f]; pDestFilterInfo->filterOnBinary = htons(pFilterInfo->filterOnBinary); @@ -5546,27 +5512,23 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTa bool hasArithmeticFunction = false; - /* - * 1. simple projection query on meters, we only record the pSqlFuncExprs[i].colIdx value - * 2. for complex queries, whole SqlExprs object is required. - */ - pQueryTableMsg->pSqlFuncExprs = (int64_t)malloc(POINTER_BYTES * pQueryTableMsg->numOfOutputCols); + *pExpr = calloc(pQueryTableMsg->numOfOutputCols, POINTER_BYTES); + SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { - ((SSqlFuncExprMsg **)pQueryTableMsg->pSqlFuncExprs)[i] = pExprMsg; + (*pExpr)[i] = pExprMsg; pExprMsg->colInfo.colIdx = htons(pExprMsg->colInfo.colIdx); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); pMsg += sizeof(SSqlFuncExprMsg); for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); + pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); pExprMsg->arg[j].argBytes = htons(pExprMsg->arg[j].argBytes); if (pExprMsg->arg[j].argType == TSDB_DATA_TYPE_BINARY) { @@ -5599,44 +5561,20 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTa pQueryTableMsg->colNameList = (int64_t)pMsg; pMsg += pQueryTableMsg->colNameLen; } + + pMsg = createTableIdList(pQueryTableMsg, pMsg, pTableIdList); - *pTableIdList = taosArrayInit(pQueryTableMsg->numOfTables, sizeof(STableIdInfo)); - - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->sid = htonl(pTableIdInfo->sid); - pTableIdInfo->uid = htobe64(pTableIdInfo->uid); - pTableIdInfo->key = htobe64(pTableIdInfo->key); - - taosArrayPush(*pTableIdList, pTableIdInfo); - pMsg += sizeof(STableIdInfo); - - for (int32_t j = 1; j < pQueryTableMsg->numOfTables; ++j) { - pTableIdInfo = (STableIdInfo *)pMsg; - - pTableIdInfo->sid = htonl(pTableIdInfo->sid); - pTableIdInfo->uid = htobe64(pTableIdInfo->uid); - pTableIdInfo->key = htobe64(pTableIdInfo->key); - - taosArrayPush(*pTableIdList, pTableIdInfo); - pMsg += sizeof(STableIdInfo); - } - - if (pQueryTableMsg->numOfGroupCols > 0 || pQueryTableMsg->numOfTagsCols > 0) { // group by tag columns - pQueryTableMsg->pTagSchema = (uint64_t)pMsg; - SSchema *pTagSchema = (SSchema *)pQueryTableMsg->pTagSchema; - pMsg += sizeof(SSchema) * pQueryTableMsg->numOfTagsCols; - - if (pQueryTableMsg->numOfGroupCols > 0) { - pQueryTableMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryTableMsg->numOfTagsCols]); - } else { - pQueryTableMsg->groupbyTagIds = 0; - } + if (pQueryTableMsg->numOfGroupCols > 0) { // group by tag columns +// if (pQueryTableMsg->numOfGroupCols > 0) { +// pQueryTableMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryTableMsg->numOfTagsCols]); +// } else { +// pQueryTableMsg->groupbyTagIds = 0; +// } pQueryTableMsg->orderByIdx = htons(pQueryTableMsg->orderByIdx); pQueryTableMsg->orderType = htons(pQueryTableMsg->orderType); pMsg += sizeof(SColIndexEx) * pQueryTableMsg->numOfGroupCols; } else { - pQueryTableMsg->pTagSchema = 0; pQueryTableMsg->groupbyTagIds = 0; } @@ -5650,15 +5588,12 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTa } } - dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 - ", numOfGroupbyTagCols:%d, numOfTagCols:%d, timestamp order:%d, " - "tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 - ", fillType:%d, comptslen:%d, limit:%" PRId64 - ", " - "offset:%" PRId64, + dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, " + "timestamp order:%d, tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 + ", fillType:%d, comptslen:%d, limit:%" PRId64 ", offset:%" PRId64, pQueryTableMsg, pQueryTableMsg->numOfTables, pQueryTableMsg->window.skey, pQueryTableMsg->window.ekey, - pQueryTableMsg->numOfGroupCols, pQueryTableMsg->numOfTagsCols, pQueryTableMsg->order, - pQueryTableMsg->orderType, pQueryTableMsg->orderByIdx, pQueryTableMsg->numOfOutputCols, + pQueryTableMsg->numOfGroupCols, pQueryTableMsg->order, pQueryTableMsg->orderType, + pQueryTableMsg->orderByIdx, pQueryTableMsg->numOfOutputCols, pQueryTableMsg->numOfCols, pQueryTableMsg->intervalTime, pQueryTableMsg->interpoType, pQueryTableMsg->tsLen, pQueryTableMsg->limit, pQueryTableMsg->offset); @@ -5716,22 +5651,20 @@ static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMs return TSDB_CODE_SUCCESS; } -static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr) { +static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr, SSqlFuncExprMsg** pExprMsg) { *pSqlFuncExpr = NULL; int32_t code = TSDB_CODE_SUCCESS; SSqlFunctionExpr *pExprs = (SSqlFunctionExpr *)calloc(1, sizeof(SSqlFunctionExpr) * pQueryMsg->numOfOutputCols); if (pExprs == NULL) { - tfree(pQueryMsg->pSqlFuncExprs); return TSDB_CODE_SERV_OUT_OF_MEMORY; } bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); int16_t tagLen = 0; - SSchema *pTagSchema = (SSchema *)pQueryMsg->pTagSchema; for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { - pExprs[i].pBase = *((SSqlFuncExprMsg **)pQueryMsg->pSqlFuncExprs)[i]; + pExprs[i].pBase = *pExprMsg[i]; pExprs[i].resBytes = 0; int16_t type = 0; @@ -5739,36 +5672,24 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct SColIndexEx *pColumnIndexExInfo = &pExprs[i].pBase.colInfo; - // tag column schema is kept in pQueryMsg->pColumnModel - if (TSDB_COL_IS_TAG(pColumnIndexExInfo->flag)) { - if (pColumnIndexExInfo->colIdx >= pQueryMsg->numOfTagsCols) { - tfree(pExprs); + // parse the arithmetic expression + if (pExprs[i].pBase.functionId == TSDB_FUNC_ARITHM) { + code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); - return TSDB_CODE_INVALID_QUERY_MSG; + if (code != TSDB_CODE_SUCCESS) { + tfree(pExprs); + return code; } - type = pTagSchema[pColumnIndexExInfo->colIdx].type; - bytes = pTagSchema[pColumnIndexExInfo->colIdx].bytes; - - } else { // parse the arithmetic expression - if (pExprs[i].pBase.functionId == TSDB_FUNC_ARITHM) { - code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); - - if (code != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return code; - } - - type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypeDesc[type].nSize; - } else { // parse the normal column - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); - assert(j < pQueryMsg->numOfCols); + type = TSDB_DATA_TYPE_DOUBLE; + bytes = tDataTypeDesc[type].nSize; + } else { // parse the normal column + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); + assert(j < pQueryMsg->numOfCols); - SColumnInfo *pCol = &pQueryMsg->colList[j]; - type = pCol->type; - bytes = pCol->bytes; - } + SColumnInfo *pCol = &pQueryMsg->colList[j]; + type = pCol->type; + bytes = pCol->bytes; } int32_t param = pExprs[i].pBase.arg[0].argValue.i64; @@ -5788,7 +5709,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct // TODO refactor for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { - pExprs[i].pBase = *((SSqlFuncExprMsg **)pQueryMsg->pSqlFuncExprs)[i]; + pExprs[i].pBase = *pExprMsg[i]; int16_t functId = pExprs[i].pBase.functionId; if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); @@ -5805,7 +5726,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct } } - tfree(pQueryMsg->pSqlFuncExprs); + tfree(pExprMsg); *pSqlFuncExpr = pExprs; return TSDB_CODE_SUCCESS; @@ -5957,8 +5878,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i].info = pQueryMsg->colList[i]; - // SColumnInfo *pColInfo = &pQuery->colList[i].data; - // pColInfo->filters = NULL; + + SColumnInfo *pColInfo = &pQuery->colList[i].info; + pColInfo->filters = NULL; // if (colList[i].numOfFilters > 0) { // pColInfo->filters = calloc(1, colList[i].numOfFilters * sizeof(SColumnFilterInfo)); // @@ -5982,7 +5904,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } // prepare the result buffer - pQuery->sdata = (SData **)calloc(pQuery->numOfOutputCols, sizeof(SData *)); + pQuery->sdata = (SData **)calloc(pQuery->numOfOutputCols, POINTER_BYTES); if (pQuery->sdata == NULL) { goto _clean_memory; } @@ -6055,21 +5977,16 @@ bool isQInfoValid(void *param) { return (sig == (uint64_t)pQInfo); } -void vnodeFreeQInfo(SQInfo *pQInfo, bool decQueryRef) { +void vnodeFreeQInfo(SQInfo *pQInfo) { if (!isQInfoValid(pQInfo)) { return; } - pQInfo->killed = 1; + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + setQueryKilled(pQInfo); + dTrace("QInfo:%p start to free SQInfo", pQInfo); - - if (decQueryRef) { - vnodeDecMeterRefcnt(pQInfo); - } - - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - for (int col = 0; col < pQuery->numOfOutputCols; ++col) { + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { tfree(pQuery->sdata[col]); } @@ -6115,7 +6032,7 @@ void vnodeFreeQInfo(SQInfo *pQInfo, bool decQueryRef) { tfree(pQuery->pGroupbyExpr); tfree(pQuery); - // dTrace("QInfo:%p vid:%d sid:%d meterId:%s, QInfo is freed", pQInfo, pObj->vnode, pObj->sid, pObj->meterId); + dTrace("QInfo:%p QInfo is freed", pQInfo); // destroy signature, in order to avoid the query process pass the object safety check memset(pQInfo, 0, sizeof(SQInfo)); @@ -6137,12 +6054,10 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE pQuery->window.skey = pQueryMsg->window.skey; pQuery->window.ekey = pQueryMsg->window.ekey; - - pQuery->lastKey = pQuery->window.skey; + pQuery->lastKey = pQuery->window.skey; if (sem_init(&(*pQInfo)->dataReady, 0, 0) != 0) { - // dError("QInfo:%p vid:%d sid:%d meterId:%s, init dataReady sem failed, reason:%s", pQInfo, pMeterObj->vnode, - // pMeterObj->sid, pMeterObj->meterId, strerror(errno)); + dError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); code = TSDB_CODE_APP_ERROR; goto _error; } @@ -6158,7 +6073,7 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE tsBufNextPos(pTSBuf); } - if ((code = vnodeQueryTablePrepare(*pQInfo, pTSBuf, tsdb)) != TSDB_CODE_SUCCESS) { + if ((code = initQInfo(*pQInfo, pTSBuf, tsdb)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -6173,7 +6088,7 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE _error: // table query ref will be decrease during error handling - vnodeFreeQInfo(*pQInfo, false); + vnodeFreeQInfo(*pQInfo); return code; } @@ -6181,8 +6096,10 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQ assert(pQueryTableMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; + SArray *pTableIdList = NULL; - if ((code = convertQueryTableMsg(pQueryTableMsg, &pTableIdList)) != TSDB_CODE_SUCCESS) { + SSqlFuncExprMsg** pExprMsg = NULL; + if ((code = convertQueryMsg(pQueryTableMsg, &pTableIdList, &pExprMsg)) != TSDB_CODE_SUCCESS) { return code; } @@ -6200,7 +6117,7 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQ } SSqlFunctionExpr *pExprs = NULL; - if ((code = createSqlFunctionExprFromMsg(pQueryTableMsg, &pExprs)) != TSDB_CODE_SUCCESS) { + if ((code = createSqlFunctionExprFromMsg(pQueryTableMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { goto _query_over; } @@ -6242,28 +6159,139 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo, int32_t *numOfRows, int32_t *ro if (pQInfo == NULL || !isQInfoValid(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } - - if (pQInfo->killed) { + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (isQueryKilled(pQuery)) { dTrace("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); if (pQInfo->code == TSDB_CODE_SUCCESS) { return TSDB_CODE_QUERY_CANCELLED; } else { // in case of not TSDB_CODE_SUCCESS, return the code to client - return abs(pQInfo->code); + return (pQInfo->code >= 0)? pQInfo->code:(-pQInfo->code); } } sem_wait(&pQInfo->dataReady); + + *numOfRows = pQInfo->rec.pointsRead; + *rowsize = pQuery->rowSize; - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + dTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, *rowsize, *numOfRows, pQInfo->code); + + return (pQInfo->code >= 0)? pQInfo->code:(-pQInfo->code); +} - // *numOfRows = pQInfo->rec.pointsRead; - // *rowsize = pQuery->rowSize; - *numOfRows = 1; +static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + /* + * get the file size and set the numOfRows to be the file size, since for tsComp query, + * the returned row size is equalled to 1 + * TODO handle the case that the file is too large to send back one time + */ + if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { + struct stat fstat; + if (stat(pQuery->sdata[0]->data, &fstat) == 0) { + *numOfRows = fstat.st_size; + return fstat.st_size; + } else { + dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); + return 0; + } + } else { + return pQuery->rowSize * (*numOfRows); + } +} - // dTrace("QInfo:%p, retrieve data info completed, precision:%d, rowsize:%d, rows:%d, code:%d", pQInfo, *timePrec, - // *rowsize, *numOfRows, pQInfo->code); +static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int32_t *size) { + // the remained number of retrieved rows, not the interpolated result + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + // load data from file to msg buffer + if (isTSCompQuery(pQuery)) { + int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); + + // make sure file exist + if (FD_VALID(fd)) { + size_t s = lseek(fd, 0, SEEK_END); + dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); + + lseek(fd, 0, SEEK_SET); + read(fd, data, s); + close(fd); + + unlink(pQuery->sdata[0]->data); + } else { + dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, + pQuery->sdata[0]->data, strerror(errno)); + } + } else { + doCopyQueryResultToMsg(pQInfo, pQInfo->rec.pointsRead, data); + } + + pQInfo->rec.pointsTotal += pQInfo->rec.pointsRead; + dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQInfo->rec.pointsRead, pQInfo->rec.pointsTotal); + + setQueryStatus(pQuery, QUERY_COMPLETED); + return TSDB_CODE_SUCCESS; + + // todo if interpolation exists, the result may be dump to client by several rounds +} - if (pQInfo->code < 0) { // less than 0 means there are error existed. - return -pQInfo->code; +static void addToTaskQueue(SQInfo* pQInfo) { + // no error occurred, continue retrieving data + if (pQInfo->code == TSDB_CODE_SUCCESS) { +#ifdef _TD_ARM_ + dTrace("QInfo:%p set query flag, sig:%" PRIu64 ", func:doDumpQueryResult", pQInfo, pQInfo->signature); +#else + dTrace("QInfo:%p set query flag, sig:%" PRIu64 ", func:%s", pQInfo, pQInfo->signature, __FUNCTION__); +#endif + + // todo add to task queue } } + +int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen) { + if (pQInfo == NULL || !isQInfoValid(pQInfo)) { + return TSDB_CODE_INVALID_QHANDLE; + } + + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + size_t size = getResultSize(pQInfo, &pQInfo->rec.pointsRead); + *contLen = size + sizeof(SRetrieveTableRsp); + + // todo handle failed to allocate memory + *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); + + (*pRsp)->numOfRows = htonl(pQInfo->rec.pointsRead); + + int32_t code = pQInfo->code; + if (code == TSDB_CODE_SUCCESS) { + (*pRsp)->offset = htobe64(pQuery->limit.offset); + (*pRsp)->useconds = htobe64(pQInfo->elapsedTime); + } else { + (*pRsp)->offset = 0; + (*pRsp)->useconds = 0; + } + + if (pQInfo->rec.pointsRead > 0 && code == TSDB_CODE_SUCCESS) { + code = doDumpQueryResult(pQInfo, (*pRsp)->data, NULL); + + // has more data to return or need next round to execute + addToTaskQueue(pQInfo); + } else if (isQueryKilled(pQuery)) { + code = TSDB_CODE_QUERY_CANCELLED; + } + + if (isQueryKilled(pQuery) || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + (*pRsp)->completed = 1; // notify no more result to client + vnodeFreeQInfo(pQInfo); + } + + return code; + +// if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { +// dTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); +// vnodeDecRefCount(pObj->qhandle); +// pObj->qhandle = NULL; +// } +} \ No newline at end of file diff --git a/src/sdb/CMakeLists.txt b/src/sdb/CMakeLists.txt deleted file mode 100644 index 47ea6e15b8cf5d8ba6f5d24c562e2b133f58f5d0..0000000000000000000000000000000000000000 --- a/src/sdb/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) -INCLUDE_DIRECTORIES(inc) - -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) - AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(sdb ${SRC}) - TARGET_LINK_LIBRARIES(sdb trpc) - IF (TD_CLUSTER) - TARGET_LINK_LIBRARIES(sdb) - ENDIF() -ENDIF () diff --git a/src/sdb/inc/sdbint.h b/src/sdb/inc/sdbint.h deleted file mode 100644 index 30ebe0909c83f5fdefd6105e31f0ecd467d1b77d..0000000000000000000000000000000000000000 --- a/src/sdb/inc/sdbint.h +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _sdbint_header_ -#define _sdbint_header_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "hashint.h" -#include "hashstr.h" -#include "sdb.h" -#include "tchecksum.h" -#include "tlog.h" -#include "trpc.h" -#include "tutil.h" - -#define sdbError(...) \ - if (sdbDebugFlag & DEBUG_ERROR) { \ - tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \ - } -#define sdbWarn(...) \ - if (sdbDebugFlag & DEBUG_WARN) { \ - tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ - } -#define sdbTrace(...) \ - if (sdbDebugFlag & DEBUG_TRACE) { \ - tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ - } -#define sdbPrint(...) \ - { tprintf("MND-SDB ", 255, __VA_ARGS__); } - -#define mpeerError(...) \ - if (sdbDebugFlag & DEBUG_ERROR) { \ - tprintf("ERROR MND-MPEER ", 255, __VA_ARGS__); \ - } -#define mpeerWarn(...) \ - if (sdbDebugFlag & DEBUG_WARN) { \ - tprintf("WARN MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \ - } -#define mpeerTrace(...) \ - if (sdbDebugFlag & DEBUG_TRACE) { \ - tprintf("MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \ - } -#define mpeerPrint(...) \ - { tprintf("MND-MPEER ", 255, __VA_ARGS__); } - -#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__) -#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__) -#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__) - -#define SDB_MAX_PEERS 4 -#define SDB_DELIMITER 0xFFF00F00 -#define SDB_ENDCOMMIT 0xAFFFAAAF - -typedef struct { - uint64_t swVersion; - int16_t sdbFileVersion; - char reserved[6]; - TSCKSUM checkSum; -} SSdbHeader; - -typedef struct { - char type; - // short rowSize; - char *row; -} SSdbUpdate; - -typedef struct _SSdbTable { - SSdbHeader header; - int maxRows; - int dbId; - int32_t maxRowSize; - char name[TSDB_DB_NAME_LEN]; - char fn[128]; - int keyType; - uint32_t autoIndex; - int64_t numOfRows; - int64_t id; - int64_t size; - void * iHandle; - int fd; - void *(*appTool)(char, void *, char *, int, int *); - pthread_mutex_t mutex; - SSdbUpdate * update; - int numOfUpdates; - int updatePos; -} SSdbTable; - -typedef struct { - int64_t id; - int64_t offset; - int rowSize; - void * row; -} SRowMeta; - -typedef struct { - int32_t delimiter; - int32_t rowSize; - int64_t id; - char data[]; -} SRowHead; - -typedef struct { - uint8_t dbId; - char type; - uint64_t version; - short dataLen; - char data[]; -} SForwardMsg; - -extern SSdbTable *tableList[]; -extern int sdbMaxPeers; -extern int sdbNumOfTables; -extern int64_t sdbVersion; - -int sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int dataLen); -int mpeerRetrieveRows(int fd, SSdbTable *pTable, uint64_t version); -void sdbResetTable(SSdbTable *pTable); -extern const int16_t sdbFileVersion; - - -#endif diff --git a/src/sdb/src/sdbEngine.c b/src/sdb/src/sdbEngine.c deleted file mode 100644 index fbc41089d1c9917aa5adae26b0052450a44e844d..0000000000000000000000000000000000000000 --- a/src/sdb/src/sdbEngine.c +++ /dev/null @@ -1,1017 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" - -#include "sdb.h" -#include "sdbint.h" -#include "tutil.h" - -#define abs(x) (((x) < 0) ? -(x) : (x)) - -extern char version[]; -const int16_t sdbFileVersion = 0; -SRpcIpSet *pSdbIpList = NULL; -SRpcIpSet *pSdbPublicIpList = NULL; -SSdbPeer * sdbPeer[SDB_MAX_PEERS]; // first slot for self - -#ifdef CLUSTER -int sdbMaster = 0; -#else -int sdbMaster = 1; -#endif - -void *(*sdbInitIndexFp[])(int maxRows, int dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash}; - -void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash}; - -void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash}; - -void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData}; - -void (*sdbCleanUpIndexFp[])(void *handle) = { - sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash, -}; - -void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = { - sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData, -}; - -SSdbTable *tableList[20]; -int sdbNumOfTables; -int64_t sdbVersion; - -int64_t sdbGetVersion() { - return sdbVersion; -}; - -int32_t sdbGetRunStatus() { - if (!tsIsCluster) { - return SDB_STATUS_SERVING; - } - - if (sdbInited == NULL) { - return SDB_STATUS_OFFLINE; - } - return sdbStatus; -} - -void sdbFinishCommit(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - uint32_t sdbEcommit = SDB_ENDCOMMIT; - - off_t offset = lseek(pTable->fd, 0, SEEK_END); - assert(offset == pTable->size); - twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - pTable->size += sizeof(sdbEcommit); -} - -int sdbOpenSdbFile(SSdbTable *pTable) { - struct stat fstat, ofstat; - uint64_t size; - char * dirc = NULL; - char * basec = NULL; - union { - char cversion[64]; - uint64_t iversion; - } swVersion; - - memcpy(swVersion.cversion, version, sizeof(uint64_t)); - - // check sdb.db and .sdb.db status - char fn[128] = "\0"; - dirc = strdup(pTable->fn); - basec = strdup(pTable->fn); - sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - tfree(dirc); - tfree(basec); - if (stat(fn, &ofstat) == 0) { // .sdb.db file exists - if (stat(pTable->fn, &fstat) == 0) { - remove(fn); - } else { - remove(pTable->fn); - rename(fn, pTable->fn); - } - } - - pTable->fd = open(pTable->fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - if (pTable->fd < 0) { - sdbError("failed to open file:%s", pTable->fn); - return -1; - } - - pTable->size = 0; - stat(pTable->fn, &fstat); - size = sizeof(pTable->header); - - if (fstat.st_size == 0) { - pTable->header.swVersion = swVersion.iversion; - pTable->header.sdbFileVersion = sdbFileVersion; - if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) { - sdbError("failed to get file header checksum, file:%s", pTable->fn); - tclose(pTable->fd); - return -1; - } - twrite(pTable->fd, &(pTable->header), size); - pTable->size += size; - sdbFinishCommit(pTable); - } else { - uint32_t sdbEcommit = 0; - off_t offset = lseek(pTable->fd, -(sizeof(sdbEcommit)), SEEK_END); - while (offset > 0) { - read(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - if (sdbEcommit == SDB_ENDCOMMIT) { - ftruncate(pTable->fd, offset + sizeof(sdbEcommit)); - break; - } - offset = lseek(pTable->fd, -(sizeof(sdbEcommit) + 1), SEEK_CUR); - } - lseek(pTable->fd, 0, SEEK_SET); - - ssize_t tsize = read(pTable->fd, &(pTable->header), size); - if (tsize < size) { - sdbError("failed to read sdb file header, file:%s", pTable->fn); - tclose(pTable->fd); - return -1; - } - - if (pTable->header.swVersion != swVersion.iversion) { - sdbWarn("sdb file:%s version not match software version", pTable->fn); - } - - if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) { - sdbError("sdb file header is broken since checksum mismatch, file:%s", pTable->fn); - tclose(pTable->fd); - return -1; - } - - pTable->size += size; - // skip end commit symbol - lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR); - pTable->size += sizeof(sdbEcommit); - } - - pTable->numOfRows = 0; - - return pTable->fd; -} - -// TODO: Change here -void sdbAddIntoUpdateList(SSdbTable *pTable, char type, char *row) { - pTable->numOfUpdates++; - pTable->updatePos = pTable->numOfUpdates % pTable->maxRows; - - if (pTable->update[pTable->updatePos].type == SDB_TYPE_DELETE) - (*(pTable->appTool))(SDB_TYPE_DESTROY, pTable->update[pTable->updatePos].row, NULL, 0, NULL); - - pTable->update[pTable->updatePos].type = type; - pTable->update[pTable->updatePos].row = row; -} - -int sdbInitTableByFile(SSdbTable *pTable) { - SRowMeta rowMeta; - int numOfDels = 0; - int bytes = 0; - int64_t oldId = 0; - void * pMetaRow = NULL; - int total_size = 0; - int real_size = 0; - int maxAutoIndex = 0; - - oldId = pTable->id; - if (sdbOpenSdbFile(pTable) < 0) return -1; - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - - sdbTrace("open sdb file:%s for read", pTable->fn); - - // Loop to read sdb file row by row - while (1) { - memset(rowHead, 0, total_size); - - bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); - if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fn); - goto sdb_exit1; - } - - if (bytes == 0) break; - - if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->size++; - lseek(pTable->fd, -(bytes - 1), SEEK_CUR); - continue; - } - - if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s, id:%d rowSize:%d maxRowSize:%d", - pTable->fn, rowHead->id, rowHead->rowSize, pTable->maxRowSize); - pTable->size += sizeof(SRowHead); - continue; - } - - // sdbTrace("%s id:%ld rowSize:%d", pTable->name, rowHead->id, - // rowHead->rowSize); - - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); - if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - // TODO: Here may cause pTable->size not end of the file - sdbError("failed to read sdb file:%s id:%d rowSize:%d", pTable->fn, rowHead->id, rowHead->rowSize); - break; - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s id:%d, skip", pTable->name, rowHead->id); - pTable->size += real_size; - continue; - } - - // Check if the the object exists already - - pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { // New object - if (rowHead->id < 0) { - /* assert(0); */ - sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); - } else { - rowMeta.id = rowHead->id; - // TODO: Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex++; - maxAutoIndex = MAX(maxAutoIndex, *(int32_t*)rowHead->data); - } - pTable->numOfRows++; - } - } else { // already exists - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex++; - maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); - } - - if (rowHead->id < 0) { // Delete the object - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL); - pTable->numOfRows--; - numOfDels++; - } else { // Reset the object TODO: is it possible to merge reset and - // update ?? - (*(pTable->appTool))(SDB_TYPE_RESET, pMetaRow, rowHead->data, rowHead->rowSize, NULL); - } - numOfDels++; - } - - pTable->size += real_size; - if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); - } - - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex = maxAutoIndex; - } - - sdbVersion += (pTable->id - oldId); - if (numOfDels > pTable->maxRows / 4) sdbSaveSnapShot(pTable); - - pTable->numOfUpdates = 0; - pTable->updatePos = 0; - - tfree(rowHead); - return 0; - -sdb_exit1: - tfree(rowHead); - return -1; -} - -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int, int *)) { - SSdbTable *pTable = (SSdbTable *)malloc(sizeof(SSdbTable)); - if (pTable == NULL) return NULL; - memset(pTable, 0, sizeof(SSdbTable)); - - int size = sizeof(SSdbUpdate) * maxRows; - pTable->update = (SSdbUpdate *)malloc(size); - if (pTable->update == NULL) { - free(pTable); - return NULL; - }; - memset(pTable->update, 0, size); - - strcpy(pTable->name, name); - pTable->keyType = keyType; - pTable->maxRows = maxRows; - pTable->maxRowSize = maxRowSize; - pTable->appTool = appTool; - sprintf(pTable->fn, "%s/%s.db", directory, pTable->name); - - if (sdbInitIndexFp[keyType] != NULL) pTable->iHandle = (*sdbInitIndexFp[keyType])(maxRows, sizeof(SRowMeta)); - - pthread_mutex_init(&pTable->mutex, NULL); - - if (sdbInitTableByFile(pTable) < 0) return NULL; - - pTable->dbId = sdbNumOfTables++; - tableList[pTable->dbId] = pTable; - - sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d", pTable->name, pTable->numOfRows, sdbNumOfTables); - - return pTable; -} - -SRowMeta *sdbGetRowMeta(void *handle, void *key) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - - if (handle == NULL) return NULL; - - pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); - - return pMeta; -} - -void *sdbGetRow(void *handle, void *key) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - - if (handle == NULL) return NULL; - - pthread_mutex_lock(&pTable->mutex); - pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); - pthread_mutex_unlock(&pTable->mutex); - - if (pMeta == NULL) return NULL; - - return pMeta->row; -} - -// row here must be encoded string (rowSize > 0) or the object it self (rowSize -// = 0) -int64_t sdbInsertRow(void *handle, void *row, int rowSize) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta rowMeta; - int64_t id = -1; - void * pObj = NULL; - int total_size = 0; - int real_size = 0; - /* char action = SDB_TYPE_INSERT; */ - - if (pTable == NULL) { - sdbError("sdb tables is null"); - return -1; - } - - if ((pTable->keyType != SDB_KEYTYPE_AUTO) || *((int64_t *)row)) - if (sdbGetRow(handle, row)) { - if (strcmp(pTable->name, "mnode") == 0) { - /* - * The first mnode created when the system just start, so the insert action may failed - * see sdbPeer.c : sdbInitPeers - */ - pTable->id++; - sdbVersion++; - sdbPrint("table:%s, record:%s already exist, think it successed, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id); - return 0; - } else { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to insert record sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); - break; - } - return -1; - } - } - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb: %s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - - if (rowSize == 0) { // object is created already - pObj = row; - } else { // encoded string, to create object - pObj = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, row, rowSize, NULL); - } - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); - - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_INSERT, rowHead->data, rowHead->rowSize) == 0) { - pTable->id++; - sdbVersion++; - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - // TODO:here need to change - *((uint32_t *)pObj) = ++pTable->autoIndex; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - // update in SDB layer - rowMeta.id = pTable->id; - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); - - /* Update the disk content */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, real_size); - pTable->size += real_size; - sdbFinishCommit(pTable); - - sdbAddIntoUpdateList(pTable, SDB_TYPE_INSERT, rowMeta.row); - - pTable->numOfRows++; - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - default: - sdbTrace("table:%s, a record is inserted, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - } - - id = rowMeta.id; - } else { - sdbError("table:%s, failed to insert record", pTable->name); - } - - tfree(rowHead); - - pthread_mutex_unlock(&pTable->mutex); - - /* callback function to update the MGMT layer */ - if (id >= 0 && pTable->appTool) (*pTable->appTool)(SDB_TYPE_INSERT, pObj, NULL, 0, NULL); - - return id; -} - -// row here can be object or null-terminated string -int sdbDeleteRow(void *handle, void *row) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int code = -1; - void * pMetaRow = NULL; - SRowHead * rowHead = NULL; - int rowSize = 0; - int total_size = 0; - /* char action = SDB_TYPE_DELETE; */ - - if (pTable == NULL) return -1; - - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, delete failed", pTable->name); - return -1; - } - - pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - rowSize = strlen((char *)row) + 1; - break; - case SDB_KEYTYPE_UINT32: - rowSize = sizeof(uint32_t); - break; - case SDB_KEYTYPE_AUTO: - rowSize = sizeof(uint64_t); - break; - default: - return -1; - } - - total_size = sizeof(SRowHead) + rowSize + sizeof(TSCKSUM); - rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_DELETE, (char *)row, rowSize) == 0) { - pTable->id++; - sdbVersion++; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->id = -(pTable->id); - memcpy(rowHead->data, row, rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, total_size); - pTable->size += total_size; - sdbFinishCommit(pTable); - - pTable->numOfRows--; - // TODO:Change the update list here - sdbAddIntoUpdateList(pTable, SDB_TYPE_DELETE, pMetaRow); - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is deleted, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, sdbVersion, pTable->id, pTable->numOfRows); - break; - } - - // Delete from current layer - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); - - code = 0; - } - - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - // callback function of the delete - if (code == 0 && pTable->appTool) (*pTable->appTool)(SDB_TYPE_DELETE, pMetaRow, NULL, 0, NULL); - - return code; -} - -// row here can be the object or the string info (encoded string) -int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int code = -1; - int total_size = 0; - int real_size = 0; - /* char action = SDB_TYPE_UPDATE; */ - - if (pTable == NULL || row == NULL) return -1; - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, (char *) row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, taosIpStr(*(int32_t *) row), sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to update record:%d, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, *(int32_t *) row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to update record, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, sdbVersion, pTable->id); - break; - } - return -1; - } - - void *pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - - if (!isUpdated) { - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, row, updateSize, NULL); // update in upper layer - } - - if (pMetaRow != row) { - memcpy(rowHead->data, row, updateSize); - rowHead->rowSize = updateSize; - } else { - (*(pTable->appTool))(SDB_TYPE_ENCODE, pMetaRow, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - ; - - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_UPDATE, rowHead->data, rowHead->rowSize) == 0) { - pTable->id++; - sdbVersion++; - - // write to the new position - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, real_size); - - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - pMeta->rowSize = rowHead->rowSize; - pTable->size += real_size; - - sdbFinishCommit(pTable); - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is updated, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, sdbVersion, - pTable->id, pTable->numOfRows); - break; - } - - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, pMetaRow); - code = 0; - } - - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - return code; -} - -// row here must be the instruction string -int sdbBatchUpdateRow(void *handle, void *row, int rowSize) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int total_size = 0; - /* char action = SDB_TYPE_BATCH_UPDATE; */ - - if (pTable == NULL || row == NULL || rowSize <= 0) return -1; - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, batch update failed", pTable->name); - return -1; - } - - void *pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - - pthread_mutex_lock(&pTable->mutex); - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_BATCH_UPDATE, row, rowSize) == 0) { - /* // write action */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - - (*(pTable->appTool))(SDB_TYPE_BEFORE_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - - void *next_row = pMetaRow; - while (next_row != NULL) { - pTable->id++; - sdbVersion++; - - void *last_row = next_row; - next_row = (*(pTable->appTool))(SDB_TYPE_BATCH_UPDATE, last_row, (char *)row, rowSize, 0); - memset(rowHead, 0, sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM)); - - // update in current layer - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - - // write to disk - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, last_row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - taosCalcChecksumAppend(0, (uint8_t *)rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pMeta->rowSize = rowHead->rowSize; - lseek(pTable->fd, pTable->size, SEEK_SET); - twrite(pTable->fd, rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pTable->size += (sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, last_row); - - if (next_row != NULL) { - pMeta = sdbGetRowMeta(handle, next_row); - } - } - - sdbFinishCommit(pTable); - - (*(pTable->appTool))(SDB_TYPE_AFTER_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - } - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - return 0; -} - -void sdbCloseTable(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - void * pNode = NULL; - void * row; - - if (pTable == NULL) return; - - while (1) { - pNode = sdbFetchRow(handle, pNode, &row); - if (row == NULL) break; - (*(pTable->appTool))(SDB_TYPE_DESTROY, row, NULL, 0, NULL); - } - - if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); - - if (pTable->fd) tclose(pTable->fd); - - pthread_mutex_destroy(&pTable->mutex); - - sdbNumOfTables--; - sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->name, pTable->id, sdbNumOfTables); - - tfree(pTable->update); - tfree(pTable); -} - -void sdbResetTable(SSdbTable *pTable) { - /* SRowHead rowHead; */ - SRowMeta rowMeta; - int bytes; - int total_size = 0; - int real_size = 0; - SRowHead *rowHead = NULL; - void * pMetaRow = NULL; - int64_t oldId = pTable->id; - int oldNumOfRows = pTable->numOfRows; - - if (sdbOpenSdbFile(pTable) < 0) return; - pTable->numOfRows = oldNumOfRows; - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory for reset, sdb:%s", pTable->name); - return; - } - - sdbPrint("open sdb file:%s for reset table", pTable->fn); - - while (1) { - memset(rowHead, 0, total_size); - - bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); - if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fn); - tfree(rowHead); - return; - } - - if (bytes == 0) break; - - if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->size++; - lseek(pTable->fd, -(bytes - 1), SEEK_CUR); - continue; - } - - if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s for reset, id:%d rowSize:%d maxRowSize:%d", - pTable->fn, rowHead->id, rowHead->rowSize, pTable->maxRowSize); - pTable->size += sizeof(SRowHead); - continue; - } - - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); - if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - sdbError("failed to read sdb file:%s for reset, id:%d rowSize:%d", pTable->fn, rowHead->id, rowHead->rowSize); - break; - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s id:%d, skip", pTable->name, rowHead->id); - pTable->size += real_size; - continue; - } - - if (abs(rowHead->id) > oldId) { // not operated - pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { // New object - if (rowHead->id < 0) { - sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); - } else { - rowMeta.id = rowHead->id; - // TODO:Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - pTable->numOfRows++; - - (*pTable->appTool)(SDB_TYPE_INSERT, rowMeta.row, NULL, 0, NULL); - } - } else { // already exists - if (rowHead->id < 0) { // Delete the object - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL); - pTable->numOfRows--; - } else { // update the object - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, rowHead->data, rowHead->rowSize, NULL); - } - } - } - - pTable->size += real_size; - if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); - } - - sdbVersion += (pTable->id - oldId); - pTable->numOfUpdates = 0; - pTable->updatePos = 0; - - tfree(rowHead); - - sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); -} - -// TODO:A problem here :use snapshot file to sync another node will cause -// problem -void sdbSaveSnapShot(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - void * pNode = NULL; - int total_size = 0; - int real_size = 0; - int size = 0; - int numOfRows = 0; - uint32_t sdbEcommit = SDB_ENDCOMMIT; - char * dirc = NULL; - char * basec = NULL; - /* char action = SDB_TYPE_INSERT; */ - - if (pTable == NULL) return; - - sdbTrace("Table:%s, save the snapshop", pTable->name); - - char fn[128] = "\0"; - dirc = strdup(pTable->fn); - basec = strdup(pTable->fn); - sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - int fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - tfree(dirc); - tfree(basec); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate memory while saving SDB snapshot, sdb:%s", pTable->name); - return; - } - memset(rowHead, 0, size); - - // Write the header - twrite(fd, &(pTable->header), sizeof(SSdbHeader)); - size += sizeof(SSdbHeader); - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - - while (1) { - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) break; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pMeta->row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while save sdb %s snapshot", pTable->name); - tfree(rowHead); - return; - } - - /* write(fd, &action, sizeof(action)); */ - /* size += sizeof(action); */ - twrite(fd, rowHead, real_size); - size += real_size; - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - numOfRows++; - } - - tfree(rowHead); - - // Remove the old file - tclose(pTable->fd); - remove(pTable->fn); - // Rename the .sdb.db file to sdb.db file - rename(fn, pTable->fn); - pTable->fd = fd; - pTable->size = size; - pTable->numOfRows = numOfRows; - - fdatasync(pTable->fd); -} - -void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - - *ppRow = NULL; - if (pTable == NULL) return NULL; - - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) return NULL; - - *ppRow = pMeta->row; - - return pNode; -} - -int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } - -int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } diff --git a/src/sdb/src/sdbstr.c b/src/sdb/src/sdbstr.c deleted file mode 100644 index 59c01eb15a95a7354b5f37acb1d521a5e89c100d..0000000000000000000000000000000000000000 --- a/src/sdb/src/sdbstr.c +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#define _DEFAULT_SOURCE -#include "sdbint.h" - -int32_t (*mpeerInitMnodesFp)(char *directory) = NULL; -void (*mpeerCleanUpMnodesFp)() = NULL; -int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; - -char *sdbStatusStr[] = { - "offline", - "unsynced", - "syncing", - "serving", - "null" -}; - -char *sdbRoleStr[] = { - "unauthed", - "undecided", - "master", - "slave", - "null" -}; - -int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { - if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(pTable, type, data, dataLen); - } else { - return 0; - } -} - -int32_t sdbInitPeers(char *directory) { - if (mpeerInitMnodesFp) { - return (*mpeerInitMnodesFp)(directory); - } else { - return 0; - } -} - -void sdbCleanUpPeers() { - if (mpeerCleanUpMnodesFp) { - (*mpeerCleanUpMnodesFp)(); - } -} diff --git a/src/sdb/inc/hashint.h b/src/util/inc/hashint.h similarity index 100% rename from src/sdb/inc/hashint.h rename to src/util/inc/hashint.h diff --git a/src/sdb/inc/hashstr.h b/src/util/inc/hashstr.h similarity index 100% rename from src/sdb/inc/hashstr.h rename to src/util/inc/hashstr.h diff --git a/src/util/inc/tglobalcfg.h b/src/util/inc/tglobalcfg.h index 993992ffcbf9a33b00761dd20141224f912db958..664eb163734eef0eccdb65a315ee89a816b9f530 100644 --- a/src/util/inc/tglobalcfg.h +++ b/src/util/inc/tglobalcfg.h @@ -103,13 +103,7 @@ extern int tsReplications; extern int tsNumOfMPeers; extern int tsMaxShellConns; -extern int tsMaxAccounts; -extern int tsMaxUsers; -extern int tsMaxDbs; extern int tsMaxTables; -extern int tsMaxDnodes; -extern int tsMaxVGroups; -extern char tsMgmtZone[]; extern char tsLocalIp[]; extern char tsDefaultDB[]; diff --git a/src/util/inc/tidpool.h b/src/util/inc/tidpool.h index 08f9d364671a5d126e699c26cb92e74365be0c78..bf352516310a1356f37359ab3052814d726ab8f0 100644 --- a/src/util/inc/tidpool.h +++ b/src/util/inc/tidpool.h @@ -34,11 +34,7 @@ void taosIdPoolCleanUp(void *handle); int taosIdPoolNumOfUsed(void *handle); -void taosIdPoolReinit(void *handle); - -void taosIdPoolMarkStatus(void *handle, int id, int status); - -void taosIdPoolSetFreeList(void *handle); +void taosIdPoolMarkStatus(void *handle, int id); #ifdef __cplusplus } diff --git a/src/util/inc/tlog.h b/src/util/inc/tlog.h index 350a2c700e0dddb15f1726eb50d47117f9228603..3a327b0a471c4671c684d80e0215f0c863239418 100644 --- a/src/util/inc/tlog.h +++ b/src/util/inc/tlog.h @@ -239,6 +239,25 @@ extern uint32_t cdebugFlag; #define monitorLWarn(...) taosLogWarn(__VA_ARGS__) monitorWarn(__VA_ARGS__) #define monitorLPrint(...) taosLogPrint(__VA_ARGS__) monitorPrint(__VA_ARGS__) +#define sdbError(...) \ + if (sdbDebugFlag & DEBUG_ERROR) { \ + tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \ + } +#define sdbWarn(...) \ + if (sdbDebugFlag & DEBUG_WARN) { \ + tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ + } +#define sdbTrace(...) \ + if (sdbDebugFlag & DEBUG_TRACE) { \ + tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ + } +#define sdbPrint(...) \ + { tprintf("MND-SDB ", 255, __VA_ARGS__); } + +#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__) +#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__) +#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__) + #ifdef __cplusplus } #endif diff --git a/src/sdb/src/hashint.c b/src/util/src/hashint.c similarity index 100% rename from src/sdb/src/hashint.c rename to src/util/src/hashint.c diff --git a/src/sdb/src/hashstr.c b/src/util/src/hashstr.c similarity index 100% rename from src/sdb/src/hashstr.c rename to src/util/src/hashstr.c diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index bbea30207cfbb275caf37673afd925bea63559d7..02de88881d9dcfdc2b0ac45114bd59036faa4d26 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -112,13 +112,7 @@ int tsReplications = TSDB_REPLICA_MIN_NUM; int tsNumOfMPeers = 3; int tsMaxShellConns = 2000; -int tsMaxAccounts = 100; -int tsMaxUsers = 1000; -int tsMaxDbs = 1000; -int tsMaxTables = 650000; -int tsMaxDnodes = 1000; -int tsMaxVGroups = 1000; -char tsMgmtZone[16] = "rzone"; +int tsMaxTables = 100000; char tsLocalIp[TSDB_IPv4ADDR_LEN] = {0}; char tsDefaultDB[TSDB_DB_NAME_LEN] = {0}; @@ -612,28 +606,10 @@ static void doInitGlobalConfig() { 1, 8640000, 0, TSDB_CFG_UTYPE_SECOND); // mgmt configs - tsInitConfigOption(cfg++, "mgmtZone", tsMgmtZone, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 0, 0, 16, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxAccounts", &tsMaxAccounts, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxUsers", &tsMaxUsers, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDbs", &tsMaxDbs, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 10000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxTables", &tsMaxTables, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 100000000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDnodes", &tsMaxDnodes, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxVGroups", &tsMaxVGroups, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 1000000, 0, TSDB_CFG_UTYPE_NONE); - + tsInitConfigOption(cfg++, "minSlidingTime", &tsMinSlidingTime, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 1000000, 0, TSDB_CFG_UTYPE_MS); diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index c50c38aa3cabd74266d4164f0ebdf4581ce670f6..5ed6b8cf232eb1fb774af75a9738f6a403cda93c 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -15,88 +15,81 @@ #include "os.h" #include "tlog.h" +#include typedef struct { int maxId; int numOfFree; int freeSlot; - int * freeList; + bool * freeList; pthread_mutex_t mutex; } id_pool_t; void *taosInitIdPool(int maxId) { - id_pool_t *pIdPool; - int * idList, i; - - if (maxId < 3) maxId = 3; - - pIdPool = (id_pool_t *)malloc(sizeof(id_pool_t)); + id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; - idList = (int *)malloc(sizeof(int) * (size_t)maxId); - if (idList == NULL) { + pIdPool->freeList = calloc(maxId, sizeof(bool)); + if (pIdPool->freeList == NULL) { free(pIdPool); return NULL; } - memset(pIdPool, 0, sizeof(id_pool_t)); pIdPool->maxId = maxId; - pIdPool->numOfFree = maxId - 1; + pIdPool->numOfFree = maxId; pIdPool->freeSlot = 0; - pIdPool->freeList = idList; pthread_mutex_init(&pIdPool->mutex, NULL); - for (i = 1; i < maxId; ++i) idList[i - 1] = i; - pTrace("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId); - return (void *)pIdPool; + return pIdPool; } int taosAllocateId(void *handle) { - id_pool_t *pIdPool; - int id = -1; - if (handle == NULL) return id; - - pIdPool = (id_pool_t *)handle; - - if (pIdPool->maxId < 3) pError("pool:%p is messed up, maxId:%d", pIdPool, pIdPool->maxId); + id_pool_t *pIdPool = handle; + if (handle == NULL) { + return -1; + } - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + int slot = -1; + pthread_mutex_lock(&pIdPool->mutex); if (pIdPool->numOfFree > 0) { - id = pIdPool->freeList[pIdPool->freeSlot]; - pIdPool->freeSlot = (pIdPool->freeSlot + 1) % pIdPool->maxId; - pIdPool->numOfFree--; + for (int i = 0; i < pIdPool->maxId; ++i) { + slot = (i + pIdPool->freeSlot) % pIdPool->maxId; + if (!pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = true; + pIdPool->freeSlot = slot + 1; + pIdPool->numOfFree--; + break; + } + } } - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); - - return id; + pthread_mutex_unlock(&pIdPool->mutex); + return slot + 1; } void taosFreeId(void *handle, int id) { - id_pool_t *pIdPool; - int slot; + id_pool_t *pIdPool = handle; + if (handle == NULL) return; - pIdPool = (id_pool_t *)handle; - if (pIdPool->freeList == NULL || pIdPool->maxId == 0) return; - if (id <= 0 || id >= pIdPool->maxId) return; - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + pthread_mutex_lock(&pIdPool->mutex); - slot = (pIdPool->freeSlot + pIdPool->numOfFree) % pIdPool->maxId; - pIdPool->freeList[slot] = id; - pIdPool->numOfFree++; + int slot = (id - 1) % pIdPool->maxId; + if (pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = false; + pIdPool->numOfFree++; + } - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + pthread_mutex_unlock(&pIdPool->mutex); } void taosIdPoolCleanUp(void *handle) { - id_pool_t *pIdPool; + id_pool_t *pIdPool = handle; - if (handle == NULL) return; - pIdPool = (id_pool_t *)handle; + if (pIdPool == NULL) return; pTrace("pool:%p is cleaned", pIdPool); @@ -110,42 +103,21 @@ void taosIdPoolCleanUp(void *handle) { } int taosIdPoolNumOfUsed(void *handle) { - id_pool_t *pIdPool = (id_pool_t *)handle; - - return pIdPool->maxId - pIdPool->numOfFree - 1; -} - -void taosIdPoolReinit(void *handle) { - id_pool_t *pIdPool; - - pIdPool = (id_pool_t *)handle; - pIdPool->numOfFree = 0; - pIdPool->freeSlot = 0; - - for (int i = 0; i < pIdPool->maxId; ++i) pIdPool->freeList[i] = 0; + id_pool_t *pIdPool = handle; + return pIdPool->maxId - pIdPool->numOfFree; } -void taosIdPoolMarkStatus(void *handle, int id, int status) { - id_pool_t *pIdPool = (id_pool_t *)handle; - - pIdPool->freeList[id] = status; -} +void taosIdPoolMarkStatus(void *handle, int id) { + id_pool_t *pIdPool = handle; + pthread_mutex_lock(&pIdPool->mutex); -void taosIdPoolSetFreeList(void *handle) { - id_pool_t *pIdPool; - int pos = 0; - - pIdPool = (id_pool_t *)handle; - pIdPool->numOfFree = 0; - pIdPool->freeSlot = 0; - - for (int i = 1; i < pIdPool->maxId; ++i) { - if (pIdPool->freeList[i] == 0) { - pIdPool->freeList[pos] = i; - pIdPool->numOfFree++; - pos++; - } + int slot = (id - 1) % pIdPool->maxId; + if (!pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = true; + pIdPool->numOfFree--; } + + pthread_mutex_unlock(&pIdPool->mutex); } int taosUpdateIdPool(id_pool_t *handle, int maxId) { @@ -154,18 +126,14 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { return -1; } - int *idList, i; - idList = (int *)malloc(sizeof(int) * (size_t)maxId); + int *idList = calloc(maxId, sizeof(bool)); if (idList == NULL) { return -1; } - for (i = 1; i < maxId; ++i) { - idList[i - 1] = i; - } - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + pthread_mutex_lock(&pIdPool->mutex); - memcpy(idList, pIdPool->freeList, sizeof(int) * (size_t)pIdPool->maxId); + memcpy(idList, pIdPool->freeList, sizeof(bool) * pIdPool->maxId); pIdPool->numOfFree += (maxId - pIdPool->maxId); pIdPool->maxId = maxId; @@ -173,7 +141,7 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { pIdPool->freeList = idList; free(oldIdList); - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + pthread_mutex_unlock(&pIdPool->mutex); return 0; } diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index d654308f9956dc34d92ba902ac9d0c087f608b6d..31cdd70f36bcfd556881b2eeab9c9d105982335f 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -109,7 +109,6 @@ typedef struct STsdbQueryHandle { uint16_t flag; // denotes reversed scan of data or not int16_t order; STimeWindow window; // the primary query time window that applies to all queries - TSKEY lastKey; int32_t blockBufferSize; SCompBlock *pBlock; int32_t numOfBlocks; @@ -264,8 +263,20 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond pQueryHandle->pColumns = pColumnInfo; pQueryHandle->loadDataAfterSeek = false; pQueryHandle->isFirstSlot = true; - - pQueryHandle->lastKey = pQueryHandle->window.skey; // ascending query + + // only support table query + assert(taosArrayGetSize(idList) == 1); + + pQueryHandle->pTableQueryInfo = calloc(1, sizeof(STableQueryRec)); + STableQueryRec* pTableQRec = pQueryHandle->pTableQueryInfo; + + pTableQRec->lastKey = pQueryHandle->window.skey; + + STableIdInfo* idInfo = taosArrayGet(pQueryHandle->pTableIdList, 0); + + STableId tableId = {.uid = idInfo->uid, .tid = idInfo->sid}; + STable *pTable = tsdbIsValidTableToInsert(tsdbGetMeta(pQueryHandle->pTsdb), tableId); + pTableQRec->pTableObj = pTable; // malloc buffer in order to load data from file int32_t numOfCols = taosArrayGetSize(pColumnInfo); @@ -295,14 +306,21 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond return (tsdb_query_handle_t)pQueryHandle; } -static int32_t next = 1; bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { - if (next == 0) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; + STable *pTable = pHandle->pTableQueryInfo->pTableObj; + + // no data in cache, abort + if (pTable->mem == NULL && pTable->imem == NULL) { + return false; + } + + // all data in mem are checked already. + if (pHandle->pTableQueryInfo->lastKey > pTable->mem->keyLast) { return false; - } else { - next = 0; - return true; } + + return true; } static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, @@ -344,9 +362,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; STableIdInfo* idInfo = taosArrayGet(pHandle->pTableIdList, 0); - STableId tableId = {.uid = idInfo->uid, .tid = idInfo->sid}; - STable *pTable = tsdbIsValidTableToInsert(tsdbGetMeta(pHandle->pTsdb), tableId); - assert(pTable != NULL); + STable *pTable = pHandle->pTableQueryInfo->pTableObj; TSKEY skey = 0, ekey = 0; int32_t rows = 0; @@ -357,12 +373,15 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { } SDataBlockInfo blockInfo = { - .uid = tableId.uid, - .sid = tableId.tid, + .uid = idInfo->uid, + .sid = idInfo->sid, .size = rows, .window = {.skey = skey, .ekey = ekey} }; + // update the last key value + pHandle->pTableQueryInfo->lastKey = ekey + 1; + return blockInfo; }