diff --git a/.gitignore b/.gitignore index ba8611030b65b2138349e920c159bc77c20864c1..f2c1cb75b3405954e97dcab3c0fade3772e968a3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ rpms/ mac/ *.pyc *.tmp +*.swp src/connector/nodejs/node_modules/ src/connector/nodejs/out/ tests/test/ diff --git a/cmake/define.inc b/cmake/define.inc index ba61d3fe9d2dba3cbaafa80bafa46a48d59c8d71..e43e57097975e6aa34298023c4ed4ca85c310120 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -3,10 +3,17 @@ PROJECT(TDengine) IF (TD_CLUSTER) ADD_DEFINITIONS(-D_CLUSTER) - ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1) +ENDIF () + +IF (TD_MPEER) + ADD_DEFINITIONS(-D_MPEER) +ENDIF () + +IF (TD_VPEER) + ADD_DEFINITIONS(-D_VPEER) + ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3) ELSE () - ADD_DEFINITIONS(-DLITE) - ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1) + ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1) ENDIF () IF (TD_ACCOUNT) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 6bec41d146ee0ad5d8cf313f9e697768310e97af..97ebac097a1853e0be407da260dcb2370cd2385e 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -132,7 +132,6 @@ void tscFieldInfoSetExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlExpr* pExpr) void tscFieldInfoSetBinExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlFunctionExpr* pExpr); void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo); -void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo); void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size); void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 9cb43a9f3666f330704d968528986b38246b7351..054b2894c5cff77b63f57b16b069aaf60e150ecc 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -357,7 +357,6 @@ typedef struct SSqlObj { char freed : 4; char listed : 4; tsem_t rspSem; - tsem_t emptyRspSem; SSqlCmd cmd; SSqlRes res; uint8_t numOfSubs; @@ -409,7 +408,7 @@ int tscProcessSql(SSqlObj *pSql); int tscRenewMeterMeta(SSqlObj *pSql, char *tableId); void tscQueueAsyncRes(SSqlObj *pSql); -void tscQueueAsyncError(void(*fp), void *param); +void tscQueueAsyncError(void(*fp), void *param, int32_t code); int tscProcessLocalCmd(SSqlObj *pSql); int tscCfgDynamicOptions(char *msg); @@ -450,7 +449,7 @@ void tscFreeSqlObj(SSqlObj *pObj); void tscCloseTscObj(STscObj *pObj); -void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, int32_t sqlLen); +void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen); void tscProcessMultiVnodesInsert(SSqlObj *pSql); void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 985a6741f95ded4e3f430e5ee656434d2f4cef42..10878ee37fc8eb8b2717973657965f3e26b1d31d 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -40,7 +40,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows); -void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, int32_t sqlLen) { +void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; @@ -51,17 +51,15 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { tscError("failed to malloc payload"); - tfree(pSql); - tscQueueAsyncError(fp, param); + tscQueueAsyncError(fp, param, TSDB_CODE_CLI_OUT_OF_MEMORY); return; } - pSql->sqlstr = malloc(sqlLen + 1); + pSql->sqlstr = realloc(pSql->sqlstr, sqlLen + 1); if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); - tscQueueAsyncError(fp, param); + tscQueueAsyncError(fp, param, TSDB_CODE_CLI_OUT_OF_MEMORY); free(pCmd->payload); - free(pSql); return; } @@ -75,7 +73,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; if (code != TSDB_CODE_SUCCESS) { - pSql->res.code = (uint8_t)code; + pSql->res.code = code; tscQueueAsyncRes(pSql); return; } @@ -88,15 +86,16 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { tscError("bug!!! pObj:%p", pObj); - globalCode = TSDB_CODE_DISCONNECTED; - tscQueueAsyncError(fp, param); + terrno = TSDB_CODE_DISCONNECTED; + tscQueueAsyncError(fp, param, TSDB_CODE_DISCONNECTED); return; } int32_t sqlLen = strlen(sqlstr); if (sqlLen > tsMaxSQLStringLen) { tscError("sql string too long"); - tscQueueAsyncError(fp, param); + terrno = TSDB_CODE_INVALID_SQL; + tscQueueAsyncError(fp, param, TSDB_CODE_INVALID_SQL); return; } @@ -105,7 +104,8 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { tscError("failed to malloc sqlObj"); - tscQueueAsyncError(fp, param); + terrno = TSDB_CODE_CLI_OUT_OF_MEMORY; + tscQueueAsyncError(fp, param, TSDB_CODE_CLI_OUT_OF_MEMORY); return; } @@ -170,7 +170,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo pRes->code = numOfRows; } - tscQueueAsyncError(pSql->fetchFp, param); + tscQueueAsyncError(pSql->fetchFp, param, pRes->code); return; } @@ -200,8 +200,8 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi SSqlObj *pSql = (SSqlObj *)taosa; if (pSql == NULL || pSql->signature != pSql) { tscError("sql object is NULL"); - globalCode = TSDB_CODE_DISCONNECTED; - tscQueueAsyncError(fp, param); +// globalCode = TSDB_CODE_DISCONNECTED; + tscQueueAsyncError(fp, param, TSDB_CODE_DISCONNECTED); return; } @@ -210,7 +210,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi if (pRes->qhandle == 0) { tscError("qhandle is NULL"); - tscQueueAsyncError(fp, param); + tscQueueAsyncError(fp, param, TSDB_CODE_INVALID_QHANDLE); return; } @@ -232,8 +232,8 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), SSqlObj *pSql = (SSqlObj *)taosa; if (pSql == NULL || pSql->signature != pSql) { tscError("sql object is NULL"); - globalCode = TSDB_CODE_DISCONNECTED; - tscQueueAsyncError(fp, param); +// globalCode = TSDB_CODE_DISCONNECTED; + tscQueueAsyncError(fp, param, TSDB_CODE_DISCONNECTED); return; } @@ -242,7 +242,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), if (pRes->qhandle == 0) { tscError("qhandle is NULL"); - tscQueueAsyncError(fp, param); + tscQueueAsyncError(fp, param, TSDB_CODE_INVALID_QHANDLE); return; } @@ -331,7 +331,7 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) { // pCmd may be released, so cache pCmd->command int cmd = pCmd->command; - int code = pRes->code ? -pRes->code : pRes->numOfRows; + int code = pRes->code;// ? -pRes->code : pRes->numOfRows; // in case of async insert, restore the user specified callback function bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); @@ -349,18 +349,20 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) { } } -void tscProcessAsyncError(SSchedMsg *pMsg) { +static void tscProcessAsyncError(SSchedMsg *pMsg) { void (*fp)() = pMsg->ahandle; - - (*fp)(pMsg->thandle, NULL, -1); + (*fp)(pMsg->thandle, NULL, *(int32_t*)pMsg->msg); } -void tscQueueAsyncError(void(*fp), void *param) { +void tscQueueAsyncError(void(*fp), void *param, int32_t code) { + int32_t* c = malloc(sizeof(int32_t)); + *c = code; + SSchedMsg schedMsg; schedMsg.fp = tscProcessAsyncError; schedMsg.ahandle = fp; schedMsg.thandle = param; - schedMsg.msg = NULL; + schedMsg.msg = c; taosScheduleTask(tscQhandle, &schedMsg); } @@ -369,7 +371,7 @@ void tscQueueAsyncRes(SSqlObj *pSql) { tscTrace("%p SqlObj is freed, not add into queue async res", pSql); return; } else { - tscError("%p add into queued async res, code:%d", pSql, pSql->res.code); + tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code)); } SSchedMsg schedMsg; @@ -410,10 +412,9 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { pSql->fp = NULL; if (code != 0) { - code = abs(code); pRes->code = code; tscTrace("%p failed to renew tableMeta", pSql); - tsem_post(&pSql->rspSem); +// tsem_post(&pSql->rspSem); } else { tscTrace("%p renew tableMeta successfully, command:%d, code:%d, retry:%d", pSql, pSql->cmd.command, pSql->res.code, pSql->retry); @@ -425,15 +426,15 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { code = tscSendMsgToServer(pSql); if (code != 0) { pRes->code = code; - tsem_post(&pSql->rspSem); +// tsem_post(&pSql->rspSem); } } return; } - if (code != 0) { - pRes->code = (uint8_t)abs(code); + if (code != TSDB_CODE_SUCCESS) { + pRes->code = code; tscQueueAsyncRes(pSql); return; } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 241f24a74777be21ffb557209f20d227ff4ef957..8b065fcf51e3115792a6312ea74e4e0a130206a9 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -488,7 +488,6 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) { } tsem_init(&pSql->rspSem, 0, 0); - tsem_init(&pSql->emptyRspSem, 0, 1); pSql->signature = pSql; pSql->pTscObj = pObj; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 2505d544e481a4ad98d212ba4908dae29add35b6..5696611387049b890eeaf726d465b67d6dbc3a4d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -117,7 +117,7 @@ static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); -static int32_t tSQLBinaryExprCreateFromSqlExpr(tSQLSyntaxNode **pExpr, tSQLExpr* pAst, int32_t* num, +static int32_t convertSyntaxTreeToExprTree(tExprNode **pExpr, tSQLExpr* pAst, int32_t* num, SColIndexEx** pColIndex, SSqlExprInfo* pExprInfo); /* @@ -215,7 +215,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pQueryInfo->numOfTables == 0) { pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); } else { - pTableMetaInfo = &pQueryInfo->pTableMetaInfo[0]; + pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; } pCmd->command = pInfo->type; @@ -1208,16 +1208,16 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel SSqlBinaryExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; - tSQLSyntaxNode* pNode = NULL; + tExprNode* pNode = NULL; SColIndexEx* pColIndex = NULL; - int32_t ret = tSQLBinaryExprCreateFromSqlExpr(&pNode, pItem->pNode, &pBinExprInfo->numOfCols, &pColIndex, &pQueryInfo->exprsInfo); + int32_t ret = convertSyntaxTreeToExprTree(&pNode, pItem->pNode, &pBinExprInfo->numOfCols, &pColIndex, &pQueryInfo->exprsInfo); if (ret != TSDB_CODE_SUCCESS) { - tSQLBinaryExprDestroy(&pNode->pExpr, NULL); + tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(pQueryInfo->msg, "invalid expression in select clause"); } - pBinExprInfo->pBinExpr = pNode->pExpr; + pBinExprInfo->pBinExpr = pNode; pBinExprInfo->pReqColumns = pColIndex; for(int32_t k = 0; k < pBinExprInfo->numOfCols; ++k) { @@ -5807,20 +5807,20 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_SUCCESS; // Does not build query message here } -static int32_t tSQLBinaryExprCreateFromSqlExpr(tSQLSyntaxNode **pExpr, tSQLExpr* pAst, int32_t* num, +static int32_t convertSyntaxTreeToExprTree(tExprNode **pExpr, tSQLExpr* pAst, int32_t* num, SColIndexEx** pColIndex, SSqlExprInfo* pExprInfo) { - tSQLSyntaxNode* pLeft = NULL; - tSQLSyntaxNode* pRight= NULL; + tExprNode* pLeft = NULL; + tExprNode* pRight= NULL; if (pAst->pLeft != NULL) { - int32_t ret = tSQLBinaryExprCreateFromSqlExpr(&pLeft, pAst->pLeft, num, pColIndex, pExprInfo); + int32_t ret = convertSyntaxTreeToExprTree(&pLeft, pAst->pLeft, num, pColIndex, pExprInfo); if (ret != TSDB_CODE_SUCCESS) { return ret; } } if (pAst->pRight != NULL) { - int32_t ret = tSQLBinaryExprCreateFromSqlExpr(&pRight, pAst->pRight, num, pColIndex, pExprInfo); + int32_t ret = convertSyntaxTreeToExprTree(&pRight, pAst->pRight, num, pColIndex, pExprInfo); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -5828,14 +5828,14 @@ static int32_t tSQLBinaryExprCreateFromSqlExpr(tSQLSyntaxNode **pExpr, tSQLExpr* if (pAst->pLeft == NULL) { if (pAst->nSQLOptr >= TK_TINYINT && pAst->nSQLOptr <= TK_DOUBLE) { - *pExpr = calloc(1, sizeof(tSQLSyntaxNode) + sizeof(tVariant)); + *pExpr = calloc(1, sizeof(tExprNode) + sizeof(tVariant)); (*pExpr)->nodeType = TSQL_NODE_VALUE; - (*pExpr)->pVal = (tVariant*) ((char*)(*pExpr) + sizeof(tSQLSyntaxNode)); + (*pExpr)->pVal = (tVariant*) ((char*)(*pExpr) + sizeof(tExprNode)); tVariantAssign((*pExpr)->pVal, &pAst->val); } else if (pAst->nSQLOptr >= TK_COUNT && pAst->nSQLOptr <= TK_AVG_IRATE) { - *pExpr = calloc(1, sizeof(tSQLSyntaxNode) + sizeof(SSchemaEx)); + *pExpr = calloc(1, sizeof(tExprNode) + sizeof(SSchemaEx)); (*pExpr)->nodeType = TSQL_NODE_COL; - (*pExpr)->pSchema = (SSchema*)((char*)(*pExpr) + sizeof(tSQLSyntaxNode)); + (*pExpr)->pSchema = (SSchema*)((char*)(*pExpr) + sizeof(tExprNode)); strncpy((*pExpr)->pSchema->name, pAst->operand.z, pAst->operand.n); // set the input column data byte and type. @@ -5850,28 +5850,21 @@ static int32_t tSQLBinaryExprCreateFromSqlExpr(tSQLSyntaxNode **pExpr, tSQLExpr* return TSDB_CODE_SUCCESS; } - (*pExpr)->colId = -1; - *pColIndex = realloc(*pColIndex, (++(*num)) * sizeof(SColIndexEx)); memset(&(*pColIndex)[(*num) - 1], 0, sizeof(SColIndexEx)); strncpy((*pColIndex)[(*num) - 1].name, pAst->operand.z, pAst->operand.n); } else { - tSQLBinaryExpr *pBinExpr = (tSQLBinaryExpr *)calloc(1, sizeof(tSQLBinaryExpr)); - pBinExpr->filterOnPrimaryKey = false; - pBinExpr->pLeft = pLeft; - pBinExpr->pRight = pRight; + *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode)); + (*pExpr)->_node.hasPK = false; + (*pExpr)->_node.pLeft = pLeft; + (*pExpr)->_node.pRight = pRight; SSQLToken t = {.type = pAst->nSQLOptr}; - pBinExpr->nSQLBinaryOptr = getBinaryExprOptr(&t); + (*pExpr)->_node.optr = getBinaryExprOptr(&t); - assert(pBinExpr->nSQLBinaryOptr != 0); + assert((*pExpr)->_node.optr != 0); - (*pExpr) = malloc(sizeof(tSQLSyntaxNode)); - (*pExpr)->nodeType = TSQL_NODE_EXPR; - (*pExpr)->pExpr = pBinExpr; - (*pExpr)->colId = -1; - - if (pBinExpr->nSQLBinaryOptr == TSDB_BINARY_OP_DIVIDE) { + if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) { if (pRight->nodeType == TSQL_NODE_VALUE) { if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i64Key == 0) { return TSDB_CODE_INVALID_SQL; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8b896fb90e42006c1382a84535bd18d6374c7c65..0c51898a79351d069452dc9e03f9759a185b4625 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -285,14 +285,16 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { pRes->rspType = rpcMsg->msgType; pRes->rspLen = rpcMsg->contLen; - char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); - if (tmp == NULL) { - pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - } else { - pRes->pRsp = tmp; - if (pRes->rspLen) { + if (pRes->rspLen > 0) { + char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); + if (tmp == NULL) { + pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + } else { + pRes->pRsp = tmp; memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen); } + } else { + pRes->pRsp = NULL; } // ignore the error information returned from mnode when set ignore flag in sql @@ -325,9 +327,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { void *taosres = tscKeepConn[pCmd->command] ? pSql : NULL; - rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows; + rpcMsg->code = pRes->code ? pRes->code : pRes->numOfRows; - tscTrace("%p Async SQL result:%s res:%p", pSql, tstrerror(pRes->code), taosres); + tscTrace("%p Async SQL result:%s res:%p", pSql, tstrerror(pRes->code), pSql); /* * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj @@ -893,11 +895,6 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SCMCreateDbMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DB; - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); - return TSDB_CODE_CLI_OUT_OF_MEMORY; - } - SCMCreateDbMsg *pCreateDbMsg = (SCMCreateDbMsg*)pCmd->payload; assert(pCmd->numOfClause == 1); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index e4b07a0f645e09892c720899d9be6518c462f844..f33e589c82164ca2b09a63b478d83c66bc58e37d 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -155,6 +155,10 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) { STscObj *pObj = (STscObj *)param; assert(pObj != NULL && pObj->pSql != NULL); + if (code < 0) { + pObj->pSql->res.code = code; + } + sem_post(&pObj->pSql->rspSem); } @@ -177,17 +181,17 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha sem_wait(&pSql->rspSem); if (pSql->res.code != TSDB_CODE_SUCCESS) { + terrno = pSql->res.code; taos_close(pObj); return NULL; } tscTrace("%p DB connection is opening", pObj); - + // version compare only requires the first 3 segments of the version string int code = taosCheckVersion(version, taos_get_server_info(pObj), 3); if (code != 0) { - pSql->res.code = code; - + terrno = code; taos_close(pObj); return NULL; } else { @@ -267,31 +271,29 @@ int taos_query_imp(STscObj *pObj, SSqlObj *pSql) { return pRes->code; } -static void syncQueryCallback(void *param, TAOS_RES *tres, int code) { - STscObj *pObj = (STscObj *)param; - assert(pObj != NULL && pObj->pSql != NULL); +static void waitForQueryRsp(void *param, TAOS_RES *tres, int code) { + assert(param != NULL); + SSqlObj *pSql = ((STscObj *)param)->pSql; - sem_post(&pObj->pSql->rspSem); + // valid error code is less than 0 + if (code < 0) { + pSql->res.code = code; + } + + sem_post(&pSql->rspSem); } int taos_query(TAOS *taos, const char *sqlstr) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { - globalCode = TSDB_CODE_DISCONNECTED; + terrno = TSDB_CODE_DISCONNECTED; return TSDB_CODE_DISCONNECTED; } - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - tscError("failed to malloc sqlObj"); - return TSDB_CODE_CLI_OUT_OF_MEMORY; - } - - pObj->pSql = pSql; - tsem_init(&pSql->rspSem, 0, 0); + SSqlObj* pSql = pObj->pSql; - int32_t sqlLen = strlen(sqlstr); - doAsyncQuery(pObj, pObj->pSql, syncQueryCallback, taos, sqlstr, sqlLen); + size_t sqlLen = strlen(sqlstr); + doAsyncQuery(pObj, pObj->pSql, waitForQueryRsp, taos, sqlstr, sqlLen); // wait for the callback function to post the semaphore sem_wait(&pSql->rspSem); @@ -649,12 +651,12 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) { return pRes->tsrow; } -static void asyncFetchCallback(void *param, TAOS_RES *tres, int numOfRows) { +static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { SSqlObj* pSql = (SSqlObj*) tres; + if (numOfRows < 0) { // set the error code pSql->res.code = -numOfRows; } - sem_post(&pSql->rspSem); } @@ -677,7 +679,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { // current data are exhausted, fetch more data if (pRes->data == NULL || (pRes->data != NULL && pRes->row >= pRes->numOfRows && pRes->completed != true && (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_METRIC || pCmd->command == TSDB_SQL_FETCH))) { - taos_fetch_rows_a(res, asyncFetchCallback, pSql->pTscObj); + taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); sem_wait(&pSql->rspSem); } @@ -754,20 +756,18 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { if (pRes == NULL || pRes->qhandle == 0) { /* Query rsp is not received from vnode, so the qhandle is NULL */ tscTrace("%p qhandle is null, abort free, fp:%p", pSql, pSql->fp); - if (pSql->fp != NULL) { - STscObj* pObj = pSql->pTscObj; - - if (pSql == pObj->pSql) { - pObj->pSql = NULL; - tscFreeSqlObj(pSql); - } - + + if (tscShouldFreeAsyncSqlObj(pSql)) { + tscFreeSqlObj(pSql); tscTrace("%p Async SqlObj is freed by app", pSql); - } else if (keepCmd) { - tscFreeSqlResult(pSql); } else { - tscFreeSqlObjPartial(pSql); + if (keepCmd) { + tscFreeSqlResult(pSql); + } else { + tscFreeSqlObjPartial(pSql); + } } + return; } @@ -793,7 +793,7 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { * be executed, and the retry efforts may result in double free the resources, e.g.,SRetrieveSupport */ if (pRes->code != TSDB_CODE_QUERY_CANCELLED && - ((pRes->numOfRows > 0 && pCmd->command < TSDB_SQL_LOCAL) || + ((pRes->numOfRows > 0 && pCmd->command < TSDB_SQL_LOCAL && pRes->completed == false) || (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows == 0 && pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL))) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; @@ -836,39 +836,37 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { } } else { // if no free resource msg is sent to vnode, we free this object immediately. - - if (pSql->fp) { + bool free = tscShouldFreeAsyncSqlObj(pSql); + if (free) { assert(pRes->numOfRows == 0 || (pCmd->command > TSDB_SQL_LOCAL)); + tscFreeSqlObj(pSql); tscTrace("%p Async sql result is freed by app", pSql); - } else if (keepCmd) { - tscFreeSqlResult(pSql); - tscTrace("%p sql result is freed while sql command is kept", pSql); } else { - tscFreeSqlObjPartial(pSql); - tscTrace("%p sql result is freed", pSql); + if (keepCmd) { + tscFreeSqlResult(pSql); + tscTrace("%p sql result is freed while sql command is kept", pSql); + } else { + tscFreeSqlObjPartial(pSql); + tscTrace("%p sql result is freed by app", pSql); + } } } } void taos_free_result(TAOS_RES *res) { taos_free_result_imp(res, 0); } +// todo should not be used in async query int taos_errno(TAOS *taos) { STscObj *pObj = (STscObj *)taos; - int code; - - if (pObj == NULL || pObj->signature != pObj) return globalCode; - if ((int8_t)(pObj->pSql->res.code) == -1) - code = TSDB_CODE_OTHERS; - else - code = pObj->pSql->res.code; + if (pObj == NULL || pObj->signature != pObj) { + return terrno; + } - return code; + return pObj->pSql->res.code; } -static bool validErrorCode(int32_t code) { return code >= TSDB_CODE_SUCCESS && code < TSDB_CODE_MAX_ERROR_CODE; } - /* * In case of invalid sql error, additional information is attached to explain * why the sql is invalid @@ -888,25 +886,19 @@ static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) { return z != NULL; } +// todo should not be used in async model char *taos_errstr(TAOS *taos) { STscObj *pObj = (STscObj *)taos; - uint8_t code; if (pObj == NULL || pObj->signature != pObj) - return (char*)tstrerror(globalCode); + return (char*)tstrerror(terrno); - SSqlObj *pSql = pObj->pSql; - - if (validErrorCode(pSql->res.code)) { - code = pSql->res.code; - } else { - code = TSDB_CODE_OTHERS; // unknown error - } - - if (hasAdditionalErrorInfo(code, &pSql->cmd)) { + SSqlObj* pSql = pObj->pSql; + + if (hasAdditionalErrorInfo(pSql->res.code, &pSql->cmd)) { return pSql->cmd.payload; } else { - return (char*)tstrerror(code); + return (char*)tstrerror(pSql->res.code); } } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 616bcbba504df2d949a4cbb3aee7ef84b7f6e082..9d172d4ea5279c648b636df0d8895c1a20bda7cd 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -124,7 +124,6 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* pSql->sqlstr = sqlstr; tsem_init(&pSql->rspSem, 0, 0); - tsem_init(&pSql->emptyRspSem, 0, 1); SSqlRes *pRes = &pSql->res; pRes->numOfRows = 1; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index b0c7b68ab4cfc263f6728d4ff8e95c224ec988cf..ca6ca369e422b448728d8761d7d5dc17e8377b0c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -392,10 +392,10 @@ void freeSubqueryObj(SSqlObj* pSql) { static void doQuitSubquery(SSqlObj* pParentSql) { freeSubqueryObj(pParentSql); - tsem_wait(&pParentSql->emptyRspSem); - tsem_wait(&pParentSql->emptyRspSem); +// tsem_wait(&pParentSql->emptyRspSem); +// tsem_wait(&pParentSql->emptyRspSem); - tsem_post(&pParentSql->rspSem); +// tsem_post(&pParentSql->rspSem); } static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSubquerySupporter* pSupporter) { @@ -567,7 +567,7 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { freeSubqueryObj(pParentSql); } - tsem_post(&pParentSql->rspSem); +// tsem_post(&pParentSql->rspSem); } else { tscTrace("%p sub:%p completed, completed:%d, total:%d", pParentSql, tres, finished, numOfTotal); } @@ -662,7 +662,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { } // wait for all subquery completed - tsem_wait(&pSql->rspSem); +// tsem_wait(&pSql->rspSem); // update the records for each subquery for(int32_t i = 0; i < pSql->numOfSubs; ++i) { @@ -797,10 +797,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { tscProcessSql(pSql); } else { // first retrieve from vnode during the secondary stage sub-query if (pParentSql->fp == NULL) { - tsem_wait(&pParentSql->emptyRspSem); - tsem_wait(&pParentSql->emptyRspSem); - - tsem_post(&pParentSql->rspSem); +// tsem_post(&pParentSql->rspSem); } else { // set the command flag must be after the semaphore been correctly set. // pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; @@ -954,10 +951,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { } } - tsem_post(&pSql->emptyRspSem); - tsem_wait(&pSql->rspSem); - - tsem_post(&pSql->emptyRspSem); +// tsem_wait(&pSql->rspSem); if (pSql->numOfSubs <= 0) { pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 7a585bfa68a3b82af98ae0ff358235f0ea9ed2a4..ead8c09dec3062b74745318b29c6eb3af11b96f8 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -40,15 +40,10 @@ void * tscQhandle; void * tscCheckDiskUsageTmr; int tsInsertHeadSize; -extern int tscEmbedded; -int tscNumOfThreads; -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -static pthread_mutex_t tscMutex; +int tscNumOfThreads; -extern int tsTscEnableRecordSql; -extern int tsNumOfLogLines; +static pthread_once_t tscinit = PTHREAD_ONCE_INIT; void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); -void deltaToUtcInitOnce(); void tscCheckDiskUsage(void *para, void *unused) { taosGetDisk(); @@ -60,7 +55,6 @@ int32_t tscInitRpc(const char *user, const char *secret) { char secretEncrypt[32] = {0}; taosEncryptPass((uint8_t *)secret, strlen(secret), secretEncrypt); - pthread_mutex_lock(&tscMutex); if (pVnodeConn == NULL) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localIp = tsLocalIp; @@ -78,7 +72,6 @@ int32_t tscInitRpc(const char *user, const char *secret) { pVnodeConn = rpcOpen(&rpcInit); if (pVnodeConn == NULL) { tscError("failed to init connection to vnode"); - pthread_mutex_unlock(&tscMutex); return -1; } } @@ -100,12 +93,10 @@ int32_t tscInitRpc(const char *user, const char *secret) { pTscMgmtConn = rpcOpen(&rpcInit); if (pTscMgmtConn == NULL) { tscError("failed to init connection to mgmt"); - pthread_mutex_unlock(&tscMutex); return -1; } } - pthread_mutex_unlock(&tscMutex); return 0; } @@ -113,7 +104,7 @@ void taos_init_imp() { char temp[128]; struct stat dirstat; - pthread_mutex_init(&tscMutex, NULL); + errno = TSDB_CODE_SUCCESS; srand(taosGetTimestampSec()); deltaToUtcInitOnce(); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ec6881db3f5ae4f1e32dbf790bfb662ae3df84c5..5c7d8789eaa54d52e34dcb1d4e4c83eae7d23bbe 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -469,7 +469,7 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) { pSql->freed = 0; tscFreeSqlCmdData(pCmd); - tscTrace("%p free sqlObj partial completed", pSql); + tscTrace("%p partially free sqlObj completed", pSql); } void tscFreeSqlObj(SSqlObj* pSql) { @@ -487,8 +487,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { tfree(pCmd->payload); pCmd->allocSize = 0; - - tsem_destroy(&pSql->rspSem); free(pSql); } @@ -820,7 +818,9 @@ void tscCloseTscObj(STscObj* pObj) { taosTmrStopA(&(pObj->pTimer)); tscFreeSqlObj(pSql); + sem_destroy(&pSql->rspSem); pthread_mutex_destroy(&pObj->mutex); + tscTrace("%p DB connection is closed", pObj); tfree(pObj); } @@ -842,10 +842,9 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) { if (pCmd->payload == NULL) { assert(pCmd->allocSize == 0); - pCmd->payload = (char*)malloc(size); + pCmd->payload = (char*)calloc(1, size); if (pCmd->payload == NULL) return TSDB_CODE_CLI_OUT_OF_MEMORY; pCmd->allocSize = size; - memset(pCmd->payload, 0, pCmd->allocSize); } else { if (pCmd->allocSize < size) { char* b = realloc(pCmd->payload, size); @@ -853,6 +852,8 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) { pCmd->payload = b; pCmd->allocSize = size; } + + memset(pCmd->payload, 0, pCmd->payloadLen); } //memset(pCmd->payload, 0, pCmd->allocSize); @@ -1105,7 +1106,7 @@ void tscClearFieldInfo(SFieldInfo* pFieldInfo) { for(int32_t i = 0; i < pFieldInfo->numOfOutputCols; ++i) { if (pFieldInfo->pExpr[i] != NULL) { - tSQLBinaryExprDestroy(&pFieldInfo->pExpr[i]->binExprInfo.pBinExpr, NULL); + tExprTreeDestroy(&pFieldInfo->pExpr[i]->binExprInfo.pBinExpr, NULL); tfree(pFieldInfo->pExpr[i]->binExprInfo.pReqColumns); tfree(pFieldInfo->pExpr[i]); } @@ -1742,7 +1743,7 @@ bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql) { } STscObj* pTscObj = pSql->pTscObj; - if (pSql->pStream != NULL || pTscObj->pHb == pSql) { + if (pSql->pStream != NULL || pTscObj->pHb == pSql || pTscObj->pSql == pSql) { return false; } @@ -1929,7 +1930,6 @@ STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST } pTableMetaInfo->pTableMeta = pTableMeta; -// pTableMetaInfo->pMetricMeta = pMetricMeta; pTableMetaInfo->numOfTags = numOfTags; if (tags != NULL) { @@ -1963,7 +1963,7 @@ void doRemoveMeterMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro } void tscRemoveAllMeterMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { - tscTrace("%p deref the metric/meter meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); + tscTrace("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); int32_t index = pQueryInfo->numOfTables; while (index >= 0) { diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index 1c3e3a8638eb30f312ac5df4035ced5e57ea2358..0a1f0159127cbcbb58ccb7b9a8b5bc13975fce96 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -286,6 +286,7 @@ static void dnodeProcessRetrieveMsg(void *pVnode, SReadMsg *pMsg) { dnodeContinueExecuteQuery(pVnode, pQInfo, pMsg); } else { // no further execution invoked, release the ref to vnode dnodeProcessReadResult(pVnode, pMsg); + dnodeReleaseVnode(pVnode); } } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index c5752c65c8f0534c6c42cdac300a5f2b3b1f01ae..21537d519b8c87d8d16ee1b82f4b8428721f2b12 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -80,7 +80,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ALREADY_EXIST, 0, 34, "table already ex TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_USER, 0, 35, "invalid user") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT, 0, 36, "invalid account") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PASS, 0, 37, "invalid password") -TAOS_DEFINE_ERROR(TSDB_CODE_DB_NOT_SELECTED, 0, 38, "do not selected") +TAOS_DEFINE_ERROR(TSDB_CODE_DB_NOT_SELECTED, 0, 38, "db not selected") TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, 0, 39, "memory corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_USER_ALREADY_EXIST, 0, 40, "user already exist") TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS, 0, 41, "no rights") @@ -118,7 +118,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MONITOR_DB_FORBIDDEN, 0, 72, "monitor db forbi TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS, 0, 73, "no disk permissions") TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED, 0, 74, "vg init failed") TAOS_DEFINE_ERROR(TSDB_CODE_DATA_ALREADY_IMPORTED, 0, 75, "data already imported") -TAOS_DEFINE_ERROR(TSDB_CODE_OPS_NOT_SUPPORT, 0, 76, "ops not support") +TAOS_DEFINE_ERROR(TSDB_CODE_OPS_NOT_SUPPORT, 0, 76, "operations not support") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID, 0, 77, "invalid query id") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID, 0, 78, "invalid stream id") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION, 0, 79, "invalid connection") diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index 8cfe3c7e235f3fabfed0ea403b428d579f2f26da..f82806a645d3243e73714a6392a35e4d4e446e92 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -30,9 +30,6 @@ bool mgmtCheckRedirect(void *handle); void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); -int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp); -int32_t mgmtRemoveMnode(uint32_t privateIp); - #ifdef __cplusplus } #endif diff --git a/src/mnode/src/mgmtBalance.c b/src/mnode/src/mgmtBalance.c index fc74075878869ffbd641a99f015b7fd2f7676dfe..3b8a6abc86703ffc21570f74211917d74146db90 100644 --- a/src/mnode/src/mgmtBalance.c +++ b/src/mnode/src/mgmtBalance.c @@ -49,6 +49,7 @@ int32_t mgmtAllocVnodes(SVgObj *pVgroup) { pVgroup->vnodeGid[0].dnodeId = pSelDnode->dnodeId; pVgroup->vnodeGid[0].privateIp = pSelDnode->privateIp; pVgroup->vnodeGid[0].publicIp = pSelDnode->publicIp; - mTrace("dnode:%d, alloc one vnode to vgroup", pSelDnode->dnodeId); + + mTrace("dnode:%d, alloc one vnode to vgroup, openVnodes:%d", pSelDnode->dnodeId, pSelDnode->openVnodes); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 65531bd36d6d638916884fcaf6c64a1ac1d9f641..d5203200eeeeedf9566d480d176cf3d02554cc21 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -29,10 +29,19 @@ static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg); static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; static void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg); +static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); + extern int32_t clusterInit(); extern void clusterCleanUp(); extern int32_t clusterGetDnodesNum(); -extern void * clusterGetNextDnode(void *pNode, SDnodeObj **pDnode); +extern void * clusterGetNextDnode(void *pNode, void **pDnode); extern SDnodeObj* clusterGetDnode(int32_t dnodeId); extern SDnodeObj* clusterGetDnodeByIp(uint32_t ip); #ifndef _CLUSTER @@ -43,7 +52,15 @@ int32_t mgmtInitDnodes() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, mgmtProcessCfgDnodeMsg); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp); mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg); - + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mgmtGetModuleMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mgmtRetrieveModules); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtGetConfigMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtRetrieveConfigs); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mgmtGetVnodeMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mgmtRetrieveVnodes); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mgmtGetDnodeMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mgmtRetrieveDnodes); + #ifdef _CLUSTER return clusterInit(); #else @@ -251,3 +268,437 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { rpcSendResponse(&rpcRsp); } + +static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); + if (pUser == NULL) return 0; + + if (strcmp(pUser->pAcct->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 16; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "private ip"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 16; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "public ip"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 10; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "open vnodes"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "total vnodes"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + +#ifdef _VPEER + pShow->bytes[cols] = 18; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "balance"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; +#endif + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = mgmtGetDnodesNum(); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + pShow->pNode = NULL; + + return 0; +} + +static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { + int32_t numOfRows = 0; + int32_t cols = 0; + SDnodeObj *pDnode = NULL; + char *pWrite; + char ipstr[32]; + + while (numOfRows < rows) { + pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + if (pDnode == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->dnodeId; + cols++; + + tinet_ntoa(ipstr, pDnode->privateIp); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, ipstr); + cols++; + + tinet_ntoa(ipstr, pDnode->publicIp); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, ipstr); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pDnode->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) ); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->openVnodes; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->numOfTotalVnodes; + cols++; + +#ifdef _VPEER + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, taosGetDnodeLbStatusStr(pDnode->lbStatus)); + cols++; +#endif + + numOfRows++; + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static bool clusterCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { + uint32_t status = pDnode->moduleStatus & (1 << moduleType); + return status > 0; +} + +static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { + int32_t cols = 0; + + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); + if (pUser == NULL) return 0; + + if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; + + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 16; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "IP"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 10; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "module type"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 10; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "module status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = 0; + SDnodeObj *pDnode = NULL; + while (1) { + pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + if (pDnode == NULL) break; + for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { + if (clusterCheckModuleInDnode(pDnode, moduleType)) { + pShow->numOfRows++; + } + } + } + + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + pShow->pNode = NULL; + + return 0; +} + +int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) { + int32_t numOfRows = 0; + SDnodeObj *pDnode = NULL; + char * pWrite; + int32_t cols = 0; + char ipstr[20]; + + while (numOfRows < rows) { + pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + if (pDnode == NULL) break; + + for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { + if (!clusterCheckModuleInDnode(pDnode, moduleType)) { + continue; + } + + cols = 0; + + tinet_ntoa(ipstr, pDnode->privateIp); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, ipstr); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, tsModule[moduleType].name); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) ); + cols++; + + numOfRows++; + } + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static bool clusterCheckConfigShow(SGlobalConfig *cfg) { + if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) + return false; + return true; +} + +static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { + int32_t cols = 0; + + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); + if (pUser == NULL) return 0; + + if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; + + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = TSDB_CFG_OPTION_LEN; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "config name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_CFG_VALUE_LEN; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "config value"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + + pShow->numOfRows = 0; + for (int32_t i = tsGlobalConfigNum - 1; i >= 0; --i) { + SGlobalConfig *cfg = tsGlobalConfig + i; + if (!clusterCheckConfigShow(cfg)) continue; + pShow->numOfRows++; + } + + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + pShow->pNode = NULL; + + return 0; +} + +static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) { + int32_t numOfRows = 0; + + for (int32_t i = tsGlobalConfigNum - 1; i >= 0 && numOfRows < rows; --i) { + SGlobalConfig *cfg = tsGlobalConfig + i; + if (!clusterCheckConfigShow(cfg)) continue; + + char *pWrite; + int32_t cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + snprintf(pWrite, TSDB_CFG_OPTION_LEN, "%s", cfg->option); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + switch (cfg->valType) { + case TSDB_CFG_VTYPE_SHORT: + snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + numOfRows++; + break; + case TSDB_CFG_VTYPE_INT: + snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + numOfRows++; + break; + case TSDB_CFG_VTYPE_UINT: + snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((uint32_t *)cfg->ptr)); + numOfRows++; + break; + case TSDB_CFG_VTYPE_FLOAT: + snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + numOfRows++; + break; + case TSDB_CFG_VTYPE_STRING: + case TSDB_CFG_VTYPE_IPSTR: + case TSDB_CFG_VTYPE_DIRECTORY: + snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%s", (char *)cfg->ptr); + numOfRows++; + break; + default: + break; + } + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { + int32_t cols = 0; + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); + if (pUser == NULL) return 0; + if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; + + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "vnode"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 12; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 12; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sync_status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + + SDnodeObj *pDnode = NULL; + if (pShow->payloadLen > 0 ) { + uint32_t ip = ip2uint(pShow->payload); + pDnode = mgmtGetDnodeByIp(ip); + if (NULL == pDnode) { + return TSDB_CODE_NODE_OFFLINE; + } + + SVnodeLoad* pVnode; + pShow->numOfRows = 0; + for (int32_t i = 0 ; i < TSDB_MAX_VNODES; i++) { + pVnode = &pDnode->vload[i]; + if (0 != pVnode->vgId) { + pShow->numOfRows++; + } + } + + pShow->pNode = pDnode; + } else { + while (true) { + pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + if (pDnode == NULL) break; + pShow->numOfRows += pDnode->openVnodes; + + if (0 == pShow->numOfRows) return TSDB_CODE_NODE_OFFLINE; + } + + pShow->pNode = NULL; + } + + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + + return 0; +} + +static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { + int32_t numOfRows = 0; + SDnodeObj *pDnode = NULL; + char * pWrite; + int32_t cols = 0; + + if (0 == rows) return 0; + + if (pShow->payloadLen) { + // output the vnodes info of the designated dnode. And output all vnodes of this dnode, instead of rows (max 100) + pDnode = (SDnodeObj *)(pShow->pNode); + if (pDnode != NULL) { + SVnodeLoad* pVnode; + for (int32_t i = 0 ; i < TSDB_MAX_VNODES; i++) { + pVnode = &pDnode->vload[i]; + if (0 == pVnode->vgId) { + continue; + } + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(uint32_t *)pWrite = pVnode->vgId; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, taosGetVnodeStatusStr(pVnode->status)); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, taosGetVnodeSyncStatusStr(pVnode->syncStatus)); + cols++; + + numOfRows++; + } + } + } else { + // TODO: output all vnodes of all dnodes + numOfRows = 0; + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} \ No newline at end of file diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index 71ce50eccbbc706e66e35387da63152433bcae5c..e9277651c7267fcb03c5535f3381251e7d709df8 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -15,122 +15,82 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "taoserror.h" +#include "tstatus.h" #include "trpc.h" #include "mgmtMnode.h" #include "mgmtSdb.h" +#include "mgmtShell.h" #include "mgmtUser.h" -int32_t (*mpeerAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; -int32_t (*mpeerRemoveMnodeFp)(uint32_t privateIp) = NULL; -int32_t (*mpeerGetMnodesNumFp)() = NULL; -void * (*mpeerGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; -int32_t (*mpeerInitMnodesFp)() = NULL; -void (*mpeerCleanUpMnodesFp)() = NULL; +#ifndef _MPEER static SMnodeObj tsMnodeObj = {0}; -static bool tsMnodeIsMaster = false; -static bool tsMnodeIsServing = false; static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static char *mgmtMnodeStatusStr[] = { - "offline", - "unsynced", - "syncing", - "serving", - "null" -}; - -static char *mgmtMnodeRoleStr[] = { - "unauthed", - "undecided", - "master", - "slave", - "null" -}; - int32_t mgmtInitMnodes() { - if (mpeerInitMnodesFp) { - return (*mpeerInitMnodesFp)(); - } else { - tsMnodeIsServing = true; - tsMnodeIsMaster = true; - return 0; - } -} - -void mgmtCleanupMnodes() { - if (mpeerCleanUpMnodesFp) { - (*mpeerCleanUpMnodesFp)(); - } -} - -bool mgmtInServerStatus() { - return tsMnodeIsServing; -} - -bool mgmtIsMaster() { - return tsMnodeIsMaster; -} - -bool mgmtCheckRedirect(void *handle) { - return false; -} - -int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp) { - if (mpeerAddMnodeFp) { - return (*mpeerAddMnodeFp)(privateIp, publicIp); - } else { - return 0; - } + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes); + + tsMnodeObj.mnodeId = 1; + tsMnodeObj.privateIp = inet_addr(tsPrivateIp); + tsMnodeObj.publicIp = inet_addr(tsPublicIp); + tsMnodeObj.createdTime = taosGetTimestampMs(); + tsMnodeObj.role = TSDB_MN_ROLE_MASTER; + tsMnodeObj.status = TSDB_MN_STATUS_SERVING; + tsMnodeObj.numOfMnodes = 1; + sprintf(tsMnodeObj.mnodeName, "%d", tsMnodeObj.mnodeId); + + return TSDB_CODE_SUCCESS; } -int32_t mgmtRemoveMnode(uint32_t privateIp) { - if (mpeerRemoveMnodeFp) { - return (*mpeerRemoveMnodeFp)(privateIp); - } else { - return 0; - } -} +void mgmtCleanupMnodes() {} +bool mgmtInServerStatus() { return tsMnodeObj.status == TSDB_MN_STATUS_SERVING; } +bool mgmtIsMaster() { return tsMnodeObj.role == TSDB_MN_ROLE_MASTER; } +bool mgmtCheckRedirect(void *handle) { return false; } static int32_t mgmtGetMnodesNum() { - if (mpeerGetMnodesNumFp) { - return (*mpeerGetMnodesNumFp)(); - } else { - return 1; - } + return 1; } -static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) { - if (mpeerGetNextMnodeFp) { - return (*mpeerGetNextMnodeFp)(pShow, pMnode); +static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { + if (*pMnode == NULL) { + *pMnode = &tsMnodeObj; } else { - if (*pMnode == NULL) { - *pMnode = &tsMnodeObj; - } else { - *pMnode = NULL; - } + *pMnode = NULL; } return *pMnode; } static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { - int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; - if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; + if (strcmp(pUser->pAcct->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; + int32_t cols = 0; SSchema *pSchema = pMeta->schema; + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + pShow->bytes[cols] = 16; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "private ip"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + pShow->bytes[cols] = 16; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "public ip"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create time"); @@ -149,12 +109,6 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 16; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "public ip"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - pMeta->numOfColumns = htons(cols); pShow->numOfColumns = cols; @@ -163,7 +117,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; } - pShow->numOfRows = mgmtGetMnodesNum(); + pShow->numOfRows = mgmtGetDnodesNum(); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->pNode = NULL; @@ -171,38 +125,42 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo } static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) { - int32_t numOfRows = 0; - int32_t cols = 0; + int32_t numOfRows = 0; + int32_t cols = 0; SMnodeObj *pMnode = NULL; - char *pWrite; - char ipstr[32]; + char *pWrite; + char ipstr[32]; while (numOfRows < rows) { - pShow->pNode = mgmtGetNextMnode(pShow, (SMnodeObj **)&pMnode); + pShow->pNode = mgmtGetNextMnode(pShow->pNode, (SMnodeObj **)&pMnode); if (pMnode == NULL) break; cols = 0; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pMnode->mnodeId; + cols++; + tinet_ntoa(ipstr, pMnode->privateIp); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; strcpy(pWrite, ipstr); cols++; + tinet_ntoa(ipstr, pMnode->publicIp); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pMnode->createdTime; + strcpy(pWrite, ipstr); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, mgmtMnodeStatusStr[pMnode->status]); + *(int64_t *)pWrite = pMnode->createdTime; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, mgmtMnodeRoleStr[pMnode->role]); + strcpy(pWrite, taosGetMnodeStatusStr(pMnode->status)); cols++; - tinet_ntoa(ipstr, pMnode->publicIp); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, ipstr); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strcpy(pWrite, taosGetMnodeRoleStr(pMnode->role)); cols++; numOfRows++; @@ -216,12 +174,14 @@ void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; - ipSet->ip[0] = htonl(inet_addr(tsMasterIp)); + ipSet->ip[0] = htonl(tsMnodeObj.privateIp); } void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; - ipSet->ip[0] = htonl(inet_addr(tsMasterIp)); -} \ No newline at end of file + ipSet->ip[0] = htonl(tsMnodeObj.publicIp); +} + +#endif \ No newline at end of file diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index b4ffd541f4065d6e9cbb9795d75108bc1e4d49b4..b84fe08546ca1be4c6c14acda0222212db3091e1 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -48,6 +48,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *queuedMsg); static void mgmtProcessRetrieveMsg(SQueuedMsg *queuedMsg); static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg); static void mgmtProcessConnectMsg(SQueuedMsg *queuedMsg); +static void mgmtProcessUseMsg(SQueuedMsg *queuedMsg); static void *tsMgmtShellRpc = NULL; static void *tsMgmtTranQhandle = NULL; @@ -60,7 +61,8 @@ int32_t mgmtInitShell() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_RETRIEVE, mgmtProcessRetrieveMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_HEARTBEAT, mgmtProcessHeartBeatMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg); - + mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mgmtProcessUseMsg); + tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT"); int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0; @@ -416,6 +418,23 @@ connect_over: rpcSendResponse(&rpcRsp); } +static void mgmtProcessUseMsg(SQueuedMsg *pMsg) { + SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; + + SCMUseDbMsg *pUseDbMsg = pMsg->pCont; + + // todo check for priority of current user + SDbObj* pDbObj = mgmtGetDb(pUseDbMsg->db); + + int32_t code = TSDB_CODE_SUCCESS; + if (pDbObj == NULL) { + code = TSDB_CODE_INVALID_DB; + } + + rpcRsp.code = code; + rpcSendResponse(&rpcRsp); +} + /** * check if we need to add mgmtProcessTableMetaMsg into tranQueue, which will be executed one-by-one. */ diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 23840c0b1d45717d05f87e6a606538c7e0902e8e..9c1ba2326f0e3071104814bcda6084c96b231d48 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -52,6 +52,14 @@ static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) { if (pVgroup->tableList) { tfree(pVgroup->tableList); } + + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); + if (pDnode) { + atomic_sub_fetch_32(&pDnode->openVnodes, 1); + } + } + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -87,6 +95,7 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { pVgroup->vnodeGid[i].privateIp = pDnode->privateIp; pVgroup->vnodeGid[i].publicIp = pDnode->publicIp; pVgroup->vnodeGid[i].vnode = pVgroup->vgId; + atomic_add_fetch_32(&pDnode->openVnodes, 1); } mgmtAddVgroupIntoDb(pVgroup); diff --git a/src/os/linux/src/tlinux.c b/src/os/linux/src/tlinux.c index bce4a8f13db7727cda101943529ebbd23481a32a..780e2903a064c45211ea3877c3fb2fef3a6c8bef 100644 --- a/src/os/linux/src/tlinux.c +++ b/src/os/linux/src/tlinux.c @@ -236,7 +236,7 @@ void *taosProcessAlarmSignal(void *tharg) { void (*callback)(int) = tharg; timer_t timerId; - struct sigevent sevent; + struct sigevent sevent = {0}; #ifdef _ALPINE sevent.sigev_notify = SIGEV_THREAD; diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index d52eb24726891a0beb3e4b3aa9c2a061cf902d80..616a2a46afb28506d639a6ea250147fc8bbcfc08 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -16,6 +16,7 @@ #ifndef TDENGINE_TAST_H #define TDENGINE_TAST_H +#include #ifdef __cplusplus extern "C" { #endif @@ -27,14 +28,14 @@ extern "C" { #include "taosdef.h" #include "tvariant.h" -struct tSQLBinaryExpr; +struct tExprNode; struct SSchema; struct tSkipList; struct tSkipListNode; enum { - TSQL_NODE_EXPR = 0x1, - TSQL_NODE_COL = 0x2, + TSQL_NODE_EXPR = 0x1, + TSQL_NODE_COL = 0x2, TSQL_NODE_VALUE = 0x4, }; @@ -60,50 +61,41 @@ typedef struct SBinaryFilterSupp { void * pExtInfo; } SBinaryFilterSupp; -typedef struct tSQLSyntaxNode { +typedef struct tExprNode { uint8_t nodeType; - int16_t colId; // for schema, the id of column union { - struct tSQLBinaryExpr *pExpr; - struct SSchema * pSchema; - tVariant * pVal; + struct { + uint8_t optr; // filter operator + uint8_t hasPK; // 0: do not contain primary filter, 1: contain + void * info; // support filter operation on this expression only available for leaf node + + struct tExprNode *pLeft; // left child pointer + struct tExprNode *pRight; // right child pointer + } _node; + struct SSchema *pSchema; + tVariant * pVal; }; -} tSQLSyntaxNode; +} tExprNode; -typedef struct tSQLBinaryExpr { - uint8_t nSQLBinaryOptr; // filter operator - uint8_t filterOnPrimaryKey; // 0: do not contain primary filter, 1: contain +void tSQLBinaryExprFromString(tExprNode **pExpr, SSchema *pSchema, int32_t numOfCols, char *src, int32_t len); - /* - * provide the information to support filter operation on this expression - * only available for leaf node - */ - void * info; - tSQLSyntaxNode *pLeft; // left child pointer - tSQLSyntaxNode *pRight; // right child pointer -} tSQLBinaryExpr; +void tSQLBinaryExprToString(tExprNode *pExpr, char *dst, int32_t *len); -typedef struct tQueryResultset { - void ** pRes; - int64_t num; -} tQueryResultset; +void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); -void tSQLBinaryExprFromString(tSQLBinaryExpr **pExpr, SSchema *pSchema, int32_t numOfCols, char *src, int32_t len); +void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); -void tSQLBinaryExprToString(tSQLBinaryExpr *pExpr, char *dst, int32_t *len); - -void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExprs, void (*fp)(void*)); - -void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); - -void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, +void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, char *, int32_t)); -void tSQLBinaryExprTrv(tSQLBinaryExpr *pExprs, int32_t *val, int16_t *ids); -void tQueryResultClean(tQueryResultset *pRes); +void tSQLBinaryExprTrv(tExprNode *pExprs, int32_t *val, int16_t *ids); uint8_t getBinaryExprOptr(SSQLToken *pToken); +SBuffer exprTreeToBinary(tExprNode* pExprTree); + +tExprNode* exprTreeFromBinary(const void* pBuf, size_t size); + #ifdef __cplusplus } #endif diff --git a/src/query/inc/qsqlparser.h b/src/query/inc/qsqlparser.h index 064ded2fad49d3001596d6e2e7ac762ce59829b9..b1799b6902d5d66df7dac792e15461337d71936c 100644 --- a/src/query/inc/qsqlparser.h +++ b/src/query/inc/qsqlparser.h @@ -13,13 +13,14 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_QASTDEF_H -#define TDENGINE_QASTDEF_H +#ifndef TDENGINE_QSQLPARSER_H +#define TDENGINE_QSQLPARSER_H #ifdef __cplusplus extern "C" { #endif +#include #include "taos.h" #include "taosmsg.h" #include "tstoken.h" @@ -187,38 +188,19 @@ typedef struct SSqlInfo { } SSqlInfo; typedef struct tSQLExpr { - /* - * for single operand: - * TK_ALL - * TK_ID - * TK_SUM - * TK_AVG - * TK_MIN - * TK_MAX - * TK_FIRST - * TK_LAST - * TK_BOTTOM - * TK_TOP - * TK_STDDEV - * TK_PERCENTILE - * - * for binary operand: - * TK_LESS - * TK_LARGE - * TK_EQUAL etc... - */ - uint32_t nSQLOptr; // TK_FUNCTION: sql function, TK_LE: less than(binary expr) + // TK_FUNCTION: sql function, TK_LE: less than(binary expr) + uint32_t nSQLOptr; // the full sql string of function(col, param), which is actually the raw // field name, since the function name is kept in nSQLOptr already - SSQLToken operand; - struct tSQLExprList *pParam; // function parameters + SSQLToken operand; + SSQLToken colInfo; // field id + tVariant val; // value only for string, float, int - SSQLToken colInfo; // field id - tVariant val; // value only for string, float, int + struct tSQLExpr *pLeft; // left child + struct tSQLExpr *pRight; // right child - struct tSQLExpr *pLeft; // left child - struct tSQLExpr *pRight; // right child + struct tSQLExprList *pParam; // function parameters } tSQLExpr; // used in select clause. select from xxx @@ -326,18 +308,20 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType); void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *pToken); - void *ParseAlloc(void *(*mallocProc)(size_t)); +// convert the sql filter expression into binary data +int32_t tSQLExprToBinary(tSQLExpr* pExpr, SStringBuilder* sb); + enum { - TSQL_NODE_TYPE_EXPR = 0x1, - TSQL_NODE_TYPE_ID = 0x2, + TSQL_NODE_TYPE_EXPR = 0x1, + TSQL_NODE_TYPE_ID = 0x2, TSQL_NODE_TYPE_VALUE = 0x4, }; #define NON_ARITHMEIC_EXPR 0 -#define NORMAL_ARITHMETIC 1 -#define AGG_ARIGHTMEIC 2 +#define NORMAL_ARITHMETIC 1 +#define AGG_ARIGHTMEIC 2 int32_t tSQLParse(SSqlInfo *pSQLInfo, const char *pSql); @@ -345,4 +329,4 @@ int32_t tSQLParse(SSqlInfo *pSQLInfo, const char *pSql); } #endif -#endif // TDENGINE_QASTDEF_H +#endif // TDENGINE_QSQLPARSER_H diff --git a/src/query/src/qast.c b/src/query/src/qast.c index c6c65eba27e0e96a7ca895a1cdeaeec6fd302029..4524c5b2492d578aad36faae1b690f201ac9398d 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "qast.h" -#include -#include -#include "../../client/inc/tschemautil.h" #include "os.h" + +#include "tutil.h" +#include "tbuffer.h" +#include "qast.h" #include "qsqlparser.h" #include "qsyntaxtreefunction.h" #include "taosdef.h" @@ -26,7 +26,10 @@ #include "tsqlfunction.h" #include "tstoken.h" #include "ttokendef.h" -#include "tutil.h" + +#include "../../client/inc/tschemautil.h" +#include "tarray.h" +#include "tskiplist.h" /* * @@ -39,22 +42,20 @@ * ver 0.3, pipeline filter in the form of: (a+2)/9 > 14 * */ +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken); +static void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); -static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken); -static void tSQLSyntaxNodeDestroy(tSQLSyntaxNode *pNode, void (*fp)(void *)); - -static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i); -static void destroySyntaxTree(tSQLSyntaxNode *); +static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i); +static void destroySyntaxTree(tExprNode *); -static uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tSQLSyntaxNode *pLeft, - const tSQLSyntaxNode *pRight); +static uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight); /* * Check the filter value type on the right hand side based on the column id on the left hand side, * the filter value type must be identical to field type for relational operation * As for binary arithmetic operation, it is not necessary to do so. */ -static void reviseBinaryExprIfNecessary(tSQLSyntaxNode **pLeft, tSQLSyntaxNode **pRight, uint8_t *optr) { +static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, uint8_t *optr) { if (*optr >= TSDB_RELATION_LESS && *optr <= TSDB_RELATION_LIKE) { // make sure that the type of data on both sides of relational comparision are identical if ((*pLeft)->nodeType == TSQL_NODE_VALUE) { @@ -79,7 +80,7 @@ static void reviseBinaryExprIfNecessary(tSQLSyntaxNode **pLeft, tSQLSyntaxNode * */ if ((*pLeft)->nodeType == TSQL_NODE_VALUE && (*pRight)->nodeType == TSQL_NODE_COL) { if (*optr >= TSDB_RELATION_LARGE && *optr <= TSDB_RELATION_LARGE_EQUAL && *optr != TSDB_RELATION_EQUAL) { - SWAP(*pLeft, *pRight, tSQLSyntaxNode *); + SWAP(*pLeft, *pRight, tExprNode *); } switch (*optr) { @@ -101,15 +102,15 @@ static void reviseBinaryExprIfNecessary(tSQLSyntaxNode **pLeft, tSQLSyntaxNode * } } -static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken) { +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken) { /* if the token is not a value, return false */ if (pToken->type == TK_RP || (pToken->type != TK_INTEGER && pToken->type != TK_FLOAT && pToken->type != TK_ID && pToken->type != TK_TBNAME && pToken->type != TK_STRING && pToken->type != TK_BOOL)) { return NULL; } - size_t nodeSize = sizeof(tSQLSyntaxNode); - tSQLSyntaxNode *pNode = NULL; + size_t nodeSize = sizeof(tExprNode); + tExprNode *pNode = NULL; if (pToken->type == TK_ID || pToken->type == TK_TBNAME) { int32_t i = 0; @@ -130,14 +131,12 @@ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols, nodeSize += sizeof(SSchema); pNode = calloc(1, nodeSize); - pNode->pSchema = (struct SSchema *)((char *)pNode + sizeof(tSQLSyntaxNode)); + pNode->pSchema = (struct SSchema *)((char *)pNode + sizeof(tExprNode)); pNode->nodeType = TSQL_NODE_COL; if (pToken->type == TK_ID) { - pNode->colId = (int16_t)pSchema[i].colId; memcpy(pNode->pSchema, &pSchema[i], sizeof(SSchema)); } else { - pNode->colId = -1; pNode->pSchema->type = TSDB_DATA_TYPE_BINARY; pNode->pSchema->bytes = TSDB_TABLE_NAME_LEN; strcpy(pNode->pSchema->name, TSQL_TBNAME_L); @@ -147,12 +146,11 @@ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols, } else { nodeSize += sizeof(tVariant); pNode = calloc(1, nodeSize); - pNode->pVal = (tVariant *)((char *)pNode + sizeof(tSQLSyntaxNode)); + pNode->pVal = (tVariant *)((char *)pNode + sizeof(tExprNode)); toTSDBType(pToken->type); tVariantCreate(pNode->pVal, pToken); pNode->nodeType = TSQL_NODE_VALUE; - pNode->colId = -1; } return pNode; @@ -194,65 +192,61 @@ uint8_t getBinaryExprOptr(SSQLToken *pToken) { } // previous generated expr is reduced as the left child -static tSQLSyntaxNode *parseRemainStr(char *pstr, tSQLBinaryExpr *pExpr, SSchema *pSchema, int32_t optr, +static tExprNode *parseRemainStr(char *pstr, tExprNode *pExpr, SSchema *pSchema, int32_t optr, int32_t numOfCols, int32_t *i) { // set the previous generated node as the left child of new root - tSQLSyntaxNode *pLeft = malloc(sizeof(tSQLSyntaxNode)); - pLeft->nodeType = TSQL_NODE_EXPR; - pLeft->pExpr = pExpr; + pExpr->nodeType = TSQL_NODE_EXPR; // remain is the right child - tSQLSyntaxNode *pRight = createSyntaxTree(pSchema, numOfCols, pstr, i); - if (pRight == NULL || (pRight->nodeType == TSQL_NODE_COL && pLeft->nodeType != TSQL_NODE_VALUE) || - (pLeft->nodeType == TSQL_NODE_VALUE && pRight->nodeType != TSQL_NODE_COL)) { - tSQLSyntaxNodeDestroy(pLeft, NULL); - tSQLSyntaxNodeDestroy(pRight, NULL); + tExprNode *pRight = createSyntaxTree(pSchema, numOfCols, pstr, i); + if (pRight == NULL || (pRight->nodeType == TSQL_NODE_COL && pExpr->nodeType != TSQL_NODE_VALUE) || + (pExpr->nodeType == TSQL_NODE_VALUE && pRight->nodeType != TSQL_NODE_COL)) { + tExprNodeDestroy(pExpr, NULL); + tExprNodeDestroy(pRight, NULL); return NULL; } - tSQLBinaryExpr *pNewExpr = (tSQLBinaryExpr *)calloc(1, sizeof(tSQLBinaryExpr)); + tExprNode *pNewExpr = (tExprNode *)calloc(1, sizeof(tExprNode)); uint8_t k = optr; - reviseBinaryExprIfNecessary(&pLeft, &pRight, &k); - pNewExpr->pLeft = pLeft; - pNewExpr->pRight = pRight; - pNewExpr->nSQLBinaryOptr = k; + reviseBinaryExprIfNecessary(&pExpr, &pRight, &k); + pNewExpr->_node.pLeft = pExpr; + pNewExpr->_node.pRight = pRight; + pNewExpr->_node.optr = k; - pNewExpr->filterOnPrimaryKey = isQueryOnPrimaryKey(pSchema[0].name, pLeft, pRight); + pNewExpr->_node.hasPK = isQueryOnPrimaryKey(pSchema[0].name, pExpr, pRight); + pNewExpr->nodeType = TSQL_NODE_EXPR; - tSQLSyntaxNode *pn = malloc(sizeof(tSQLSyntaxNode)); - pn->nodeType = TSQL_NODE_EXPR; - pn->pExpr = pNewExpr; - - return pn; + return pNewExpr; } -uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tSQLSyntaxNode *pLeft, const tSQLSyntaxNode *pRight) { +uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) { if (pLeft->nodeType == TSQL_NODE_COL) { // if left node is the primary column,return true return (strcmp(primaryColumnName, pLeft->pSchema->name) == 0) ? 1 : 0; } else { // if any children have query on primary key, their parents are also keep this value - return ((pLeft->nodeType == TSQL_NODE_EXPR && pLeft->pExpr->filterOnPrimaryKey == 1) || - (pRight->nodeType == TSQL_NODE_EXPR && pRight->pExpr->filterOnPrimaryKey == 1)) == true + return ((pLeft->nodeType == TSQL_NODE_EXPR && pLeft->_node.hasPK == 1) || + (pRight->nodeType == TSQL_NODE_EXPR && pRight->_node.hasPK == 1)) == true ? 1 : 0; } } -static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { +static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0) { return NULL; } - tSQLSyntaxNode *pLeft = NULL; + tExprNode *pLeft = NULL; if (t0.type == TK_LP) { // start new left child branch pLeft = createSyntaxTree(pSchema, numOfCols, str, i); } else { if (t0.type == TK_RP) { return NULL; } - pLeft = tSQLSyntaxNodeCreate(pSchema, numOfCols, &t0); + + pLeft = tExprNodeCreate(pSchema, numOfCols, &t0); } if (pLeft == NULL) { @@ -262,7 +256,7 @@ static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, cha t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0 || t0.type == TK_RP) { if (pLeft->nodeType != TSQL_NODE_EXPR) { // if left is not the expr, it is not a legal expr - tSQLSyntaxNodeDestroy(pLeft, NULL); + tExprNodeDestroy(pLeft, NULL); return NULL; } @@ -273,12 +267,12 @@ static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, cha uint8_t optr = getBinaryExprOptr(&t0); if (optr == 0) { pError("not support binary operator:%d", t0.type); - tSQLSyntaxNodeDestroy(pLeft, NULL); + tExprNodeDestroy(pLeft, NULL); return NULL; } assert(pLeft != NULL); - tSQLSyntaxNode *pRight = NULL; + tExprNode *pRight = NULL; if (t0.type == TK_AND || t0.type == TK_OR || t0.type == TK_LP) { pRight = createSyntaxTree(pSchema, numOfCols, str, i); @@ -291,72 +285,68 @@ static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, cha */ t0 = tStrGetToken(str, i, true, 0, NULL); if (t0.n == 0) { - tSQLSyntaxNodeDestroy(pLeft, NULL); // illegal expression + tExprNodeDestroy(pLeft, NULL); // illegal expression return NULL; } if (t0.type == TK_LP) { pRight = createSyntaxTree(pSchema, numOfCols, str, i); } else { - pRight = tSQLSyntaxNodeCreate(pSchema, numOfCols, &t0); + pRight = tExprNodeCreate(pSchema, numOfCols, &t0); } } if (pRight == NULL) { - tSQLSyntaxNodeDestroy(pLeft, NULL); + tExprNodeDestroy(pLeft, NULL); return NULL; } /* create binary expr as the child of new parent node */ - tSQLBinaryExpr *pBinExpr = (tSQLBinaryExpr *)calloc(1, sizeof(tSQLBinaryExpr)); + tExprNode *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode)); reviseBinaryExprIfNecessary(&pLeft, &pRight, &optr); - pBinExpr->filterOnPrimaryKey = isQueryOnPrimaryKey(pSchema[0].name, pLeft, pRight); - pBinExpr->pLeft = pLeft; - pBinExpr->pRight = pRight; - pBinExpr->nSQLBinaryOptr = optr; + pExpr->_node.hasPK = isQueryOnPrimaryKey(pSchema[0].name, pLeft, pRight); + pExpr->_node.pLeft = pLeft; + pExpr->_node.pRight = pRight; + pExpr->_node.optr = optr; t0 = tStrGetToken(str, i, true, 0, NULL); if (t0.n == 0 || t0.type == TK_RP) { - tSQLSyntaxNode *pn = malloc(sizeof(tSQLSyntaxNode)); - pn->nodeType = TSQL_NODE_EXPR; - pn->pExpr = pBinExpr; - pn->colId = -1; - return pn; + pExpr->nodeType = TSQL_NODE_EXPR; + return pExpr; } else { uint8_t localOptr = getBinaryExprOptr(&t0); if (localOptr == 0) { pError("not support binary operator:%d", t0.type); - free(pBinExpr); + free(pExpr); return NULL; } - return parseRemainStr(str, pBinExpr, pSchema, localOptr, numOfCols, i); + return parseRemainStr(str, pExpr, pSchema, localOptr, numOfCols, i); } } -void tSQLBinaryExprFromString(tSQLBinaryExpr **pExpr, SSchema *pSchema, int32_t numOfCols, char *src, int32_t len) { +void tSQLBinaryExprFromString(tExprNode **pExpr, SSchema *pSchema, int32_t numOfCols, char *src, int32_t len) { *pExpr = NULL; + if (len <= 0 || src == NULL || pSchema == NULL || numOfCols <= 0) { return; } int32_t pos = 0; - tSQLSyntaxNode *pStxNode = createSyntaxTree(pSchema, numOfCols, src, &pos); - if (pStxNode != NULL) { - assert(pStxNode->nodeType == TSQL_NODE_EXPR); - *pExpr = pStxNode->pExpr; - free(pStxNode); + *pExpr = createSyntaxTree(pSchema, numOfCols, src, &pos); + if (*pExpr != NULL) { + assert((*pExpr)->nodeType == TSQL_NODE_EXPR); } } -int32_t tSQLBinaryExprToStringImpl(tSQLSyntaxNode *pNode, char *dst, uint8_t type) { +int32_t tSQLBinaryExprToStringImpl(tExprNode *pNode, char *dst, uint8_t type) { int32_t len = 0; if (type == TSQL_NODE_EXPR) { *dst = '('; - tSQLBinaryExprToString(pNode->pExpr, dst + 1, &len); + tSQLBinaryExprToString(pNode, dst + 1, &len); len += 2; *(dst + len - 1) = ')'; } else if (type == TSQL_NODE_COL) { @@ -418,49 +408,58 @@ static char *tSQLOptrToString(uint8_t optr, char *dst) { return dst; } -void tSQLBinaryExprToString(tSQLBinaryExpr *pExpr, char *dst, int32_t *len) { +void tSQLBinaryExprToString(tExprNode *pExpr, char *dst, int32_t *len) { if (pExpr == NULL) { *dst = 0; *len = 0; return; } - int32_t lhs = tSQLBinaryExprToStringImpl(pExpr->pLeft, dst, pExpr->pLeft->nodeType); + int32_t lhs = tSQLBinaryExprToStringImpl(pExpr->_node.pLeft, dst, pExpr->_node.pLeft->nodeType); dst += lhs; *len = lhs; - char *start = tSQLOptrToString(pExpr->nSQLBinaryOptr, dst); + char *start = tSQLOptrToString(pExpr->_node.optr, dst); *len += (start - dst); - *len += tSQLBinaryExprToStringImpl(pExpr->pRight, start, pExpr->pRight->nodeType); + *len += tSQLBinaryExprToStringImpl(pExpr->_node.pRight, start, pExpr->_node.pRight->nodeType); } -static void UNUSED_FUNC destroySyntaxTree(tSQLSyntaxNode *pNode) { tSQLSyntaxNodeDestroy(pNode, NULL); } +static void UNUSED_FUNC destroySyntaxTree(tExprNode *pNode) { tExprNodeDestroy(pNode, NULL); } -static void tSQLSyntaxNodeDestroy(tSQLSyntaxNode *pNode, void (*fp)(void *)) { +static void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { if (pNode == NULL) { return; } if (pNode->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprDestroy(&pNode->pExpr, fp); + tExprTreeDestroy(&pNode, fp); } else if (pNode->nodeType == TSQL_NODE_VALUE) { tVariantDestroy(pNode->pVal); + } else if (pNode->nodeType == TSQL_NODE_COL) { + free(pNode->pSchema); } free(pNode); } -void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExpr, void (*fp)(void *)) { +void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { if (*pExpr == NULL) { return; } - - tSQLSyntaxNodeDestroy((*pExpr)->pLeft, fp); - tSQLSyntaxNodeDestroy((*pExpr)->pRight, fp); - - if (fp != NULL) { - fp((*pExpr)->info); + + if ((*pExpr)->nodeType == TSQL_NODE_EXPR) { + tExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); + tExprTreeDestroy(&(*pExpr)->_node.pRight, fp); + + if (fp != NULL) { + fp((*pExpr)->_node.info); + } + } else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) { + tVariantDestroy((*pExpr)->pVal); + free((*pExpr)->pVal); + } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { + free((*pExpr)->pSchema); } free(*pExpr); @@ -571,7 +570,7 @@ void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExpr, void (*fp)(void *)) { // DEFAULT_COMP(p1, p2); //} -int32_t merge(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResultset *pFinalRes) { +int32_t merge(SArray *pLeft, SArray *pRight, SArray *pFinalRes) { // assert(pFinalRes->pRes == 0); // // pFinalRes->pRes = calloc((size_t)(pLeft->num + pRight->num), POINTER_BYTES); @@ -612,7 +611,7 @@ int32_t merge(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResultset * return 0; } -int32_t intersect(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResultset *pFinalRes) { +int32_t intersect(SArray *pLeft, SArray *pRight, SArray *pFinalRes) { // int64_t num = MIN(pLeft->num, pRight->num); // // assert(pFinalRes->pRes == 0); @@ -650,43 +649,44 @@ int32_t intersect(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResults /* * traverse the result and apply the function to each item to check if the item is qualified or not */ -static UNUSED_FUNC void tSQLListTraverseOnResult(struct tSQLBinaryExpr *pExpr, __result_filter_fn_t fp, tQueryResultset *pResult) { - assert(pExpr->pLeft->nodeType == TSQL_NODE_COL && pExpr->pRight->nodeType == TSQL_NODE_VALUE); - - // brutal force scan the result list and check for each item in the list - int64_t num = pResult->num; - for (int32_t i = 0, j = 0; i < pResult->num; ++i) { - if (fp == NULL || (fp(pResult->pRes[i], pExpr->info) == true)) { - pResult->pRes[j++] = pResult->pRes[i]; - } else { - num--; - } - } - - pResult->num = num; +static UNUSED_FUNC void tSQLListTraverseOnResult(struct tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) { +// assert(pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_VALUE); +// +// // brutal force scan the result list and check for each item in the list +// int64_t num = pResult->num; +// for (int32_t i = 0, j = 0; i < pResult->num; ++i) { +// if (fp == NULL || (fp(pResult->pRes[i], pExpr->_node.info) == true)) { +// pResult->pRes[j++] = pResult->pRes[i]; +// } else { +// num--; +// } +// } +// +// pResult->num = num; + assert(0); } -static bool filterItem(tSQLBinaryExpr *pExpr, const void *pItem, SBinaryFilterSupp *param) { - tSQLSyntaxNode *pLeft = pExpr->pLeft; - tSQLSyntaxNode *pRight = pExpr->pRight; +static bool filterItem(tExprNode *pExpr, const void *pItem, SBinaryFilterSupp *param) { + tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pRight = pExpr->_node.pRight; /* * non-leaf nodes, recursively traverse the syntax tree in the post-root order */ if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { - if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or - if (filterItem(pLeft->pExpr, pItem, param)) { + if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + if (filterItem(pLeft, pItem, param)) { return true; } // left child does not satisfy the query condition, try right child - return filterItem(pRight->pExpr, pItem, param); + return filterItem(pRight, pItem, param); } else { // and - if (!filterItem(pLeft->pExpr, pItem, param)) { + if (!filterItem(pLeft, pItem, param)) { return false; } - return filterItem(pRight->pExpr, pItem, param); + return filterItem(pRight, pItem, param); } } @@ -694,7 +694,7 @@ static bool filterItem(tSQLBinaryExpr *pExpr, const void *pItem, SBinaryFilterSu assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); param->setupInfoFn(pExpr, param->pExtInfo); - return param->fp(pItem, pExpr->info); + return param->fp(pItem, pExpr->_node.info); } /** @@ -707,7 +707,7 @@ static bool filterItem(tSQLBinaryExpr *pExpr, const void *pItem, SBinaryFilterSu * @param pSchema tag schemas * @param fp filter callback function */ -static void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, SArray *pResult, SBinaryFilterSupp *param) { +static void tSQLBinaryTraverseOnResult(tExprNode *pExpr, SArray *pResult, SBinaryFilterSupp *param) { size_t size = taosArrayGetSize(pResult); SArray* array = taosArrayInit(size, POINTER_BYTES); @@ -722,7 +722,7 @@ static void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, SArray *pResult, S taosArrayCopy(pResult, array); } -static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, SArray *pResult, SSkipList *pSkipList, +static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SBinaryFilterSupp *param) { SSkipListIterator* iter = tSkipListCreateIter(pSkipList); @@ -736,17 +736,17 @@ static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, SArray *pResult, } // post-root order traverse syntax tree -void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { +void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { if (pExpr == NULL) { return; } - tSQLSyntaxNode *pLeft = pExpr->pLeft; - tSQLSyntaxNode *pRight = pExpr->pRight; + tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pRight = pExpr->_node.pRight; // recursive traverse left child branch if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { - uint8_t weight = pLeft->pExpr->filterOnPrimaryKey + pRight->pExpr->filterOnPrimaryKey; + uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; if (weight == 0 && taosArrayGetSize(result) > 0 && pSkipList == NULL) { /** @@ -762,23 +762,23 @@ void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray */ assert(taosArrayGetSize(result) == 0); tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - } else if (weight == 2 || (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_OR)) { - tQueryResultset rLeft = {0}; - tQueryResultset rRight = {0}; + } else if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { + SArray* rLeft = taosArrayInit(10, POINTER_BYTES); + SArray* rRight = taosArrayInit(10, POINTER_BYTES); - tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, &rLeft, param); - tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, &rRight, param); + tSQLBinaryExprTraverse(pLeft, pSkipList, rLeft, param); + tSQLBinaryExprTraverse(pRight, pSkipList, rRight, param); - if (pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS - intersect(&rLeft, &rRight, result); - } else if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or - merge(&rLeft, &rRight, result); + if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS + intersect(rLeft, rRight, result); + } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + merge(rLeft, rRight, result); } else { assert(false); } - free(rLeft.pRes); - free(rRight.pRes); + taosArrayDestroy(rLeft); + taosArrayDestroy(rRight); } else { /* * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here @@ -786,16 +786,16 @@ void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray * first, we filter results based on the skiplist index, which is the initial filter stage, * then, we conduct the secondary filter operation based on the result from the initial filter stage. */ - assert(pExpr->nSQLBinaryOptr == TSDB_RELATION_AND); + assert(pExpr->_node.optr == TSDB_RELATION_AND); - tSQLBinaryExpr *pFirst = NULL; - tSQLBinaryExpr *pSecond = NULL; - if (pLeft->pExpr->filterOnPrimaryKey == 1) { - pFirst = pLeft->pExpr; - pSecond = pRight->pExpr; + tExprNode *pFirst = NULL; + tExprNode *pSecond = NULL; + if (pLeft->_node.hasPK == 1) { + pFirst = pLeft; + pSecond = pRight; } else { - pFirst = pRight->pExpr; - pSecond = pLeft->pExpr; + pFirst = pRight; + pSecond = pLeft; } assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); @@ -822,25 +822,25 @@ void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray } } -void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, +void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, char *, int32_t)) { if (pExprs == NULL) { return; } - tSQLSyntaxNode *pLeft = pExprs->pLeft; - tSQLSyntaxNode *pRight = pExprs->pRight; + tExprNode *pLeft = pExprs->_node.pLeft; + tExprNode *pRight = pExprs->_node.pRight; /* the left output has result from the left child syntax tree */ char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows); if (pLeft->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprCalcTraverse(pLeft->pExpr, numOfRows, pLeftOutput, param, order, getSourceDataBlock); + tSQLBinaryExprCalcTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); } /* the right output has result from the right child syntax tree */ char *pRightOutput = malloc(sizeof(int64_t) * numOfRows); if (pRight->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprCalcTraverse(pRight->pExpr, numOfRows, pRightOutput, param, order, getSourceDataBlock); + tSQLBinaryExprCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); } if (pLeft->nodeType == TSQL_NODE_EXPR) { @@ -849,51 +849,51 @@ void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char * exprLeft + exprRight * the type of returned value of one expression is always double float precious */ - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->nSQLBinaryOptr); + _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_COL) { // exprLeft + columnRight - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->nSQLBinaryOptr); + _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr); // set input buffer - char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->colId); + char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); fp(pLeftOutput, pInputData, numOfRows, numOfRows, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_VALUE) { // exprLeft + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->nSQLBinaryOptr); + _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr); fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, order); } } else if (pLeft->nodeType == TSQL_NODE_COL) { // column data specified on left-hand-side - char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->colId); + char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId); if (pRight->nodeType == TSQL_NODE_EXPR) { // columnLeft + expr2 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->nSQLBinaryOptr); + _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); fp(pLeftInputData, pRightOutput, numOfRows, numOfRows, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_COL) { // columnLeft + columnRight // column data specified on right-hand-side - char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->colId); + char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->nSQLBinaryOptr); + _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr); fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_VALUE) { // columnLeft + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->nSQLBinaryOptr); + _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr); fp(pLeftInputData, &pRight->pVal->i64Key, numOfRows, 1, pOutput, order); } } else { // column data specified on left-hand-side if (pRight->nodeType == TSQL_NODE_EXPR) { // 12 + expr2 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->nSQLBinaryOptr); + _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_COL) { // 12 + columnRight // column data specified on right-hand-side - char * pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->nSQLBinaryOptr); + char * pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); + _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr); fp(&pLeft->pVal->i64Key, pRightInputData, 1, numOfRows, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_VALUE) { // 12 + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->nSQLBinaryOptr); + _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr); fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, order); } } @@ -902,35 +902,121 @@ void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char free(pRightOutput); } -void tSQLBinaryExprTrv(tSQLBinaryExpr *pExprs, int32_t *val, int16_t *ids) { +void tSQLBinaryExprTrv(tExprNode *pExprs, int32_t *val, int16_t *ids) { if (pExprs == NULL) { return; } - tSQLSyntaxNode *pLeft = pExprs->pLeft; - tSQLSyntaxNode *pRight = pExprs->pRight; + tExprNode *pLeft = pExprs->_node.pLeft; + tExprNode *pRight = pExprs->_node.pRight; // recursive traverse left child branch if (pLeft->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprTrv(pLeft->pExpr, val, ids); + tSQLBinaryExprTrv(pLeft, val, ids); } else if (pLeft->nodeType == TSQL_NODE_COL) { ids[*val] = pLeft->pSchema->colId; (*val) += 1; } if (pRight->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprTrv(pRight->pExpr, val, ids); + tSQLBinaryExprTrv(pRight, val, ids); } else if (pRight->nodeType == TSQL_NODE_COL) { ids[*val] = pRight->pSchema->colId; (*val) += 1; } } -void tQueryResultClean(tQueryResultset *pRes) { - if (pRes == NULL) { - return; +static int32_t exprTreeToBinaryImpl(tExprNode* pExprTree, SBuffer* pBuf) { + tbufWrite(pBuf, &pExprTree->nodeType, sizeof(pExprTree->nodeType)); + + if (pExprTree->nodeType == TSQL_NODE_VALUE) { + tVariant* pVal = pExprTree->pVal; + + tbufWrite(pBuf, &pVal->nType, sizeof(pVal->nType)); + if (pVal->nType == TSDB_DATA_TYPE_BINARY) { + tbufWrite(pBuf, &pVal->nLen, sizeof(pVal->nLen)); + tbufWrite(pBuf, pVal->pz, pVal->nLen); + } else { + tbufWrite(pBuf, &pVal->pz, sizeof(pVal->i64Key)); + } + + } else if (pExprTree->nodeType == TSQL_NODE_COL) { + SSchema* pSchema = pExprTree->pSchema; + tbufWrite(pBuf, &pSchema->colId, sizeof(pSchema->colId)); + tbufWrite(pBuf, &pSchema->bytes, sizeof(pSchema->bytes)); + tbufWrite(pBuf, &pSchema->type, sizeof(pSchema->type)); + + int32_t len = strlen(pSchema->name); + tbufWriteStringLen(pBuf, pSchema->name, len); + + } else if (pExprTree->nodeType == TSQL_NODE_EXPR) { + tbufWrite(pBuf, &pExprTree->_node.optr, sizeof(pExprTree->_node.optr)); + tbufWrite(pBuf, &pExprTree->_node.hasPK, sizeof(pExprTree->_node.hasPK)); + + exprTreeToBinaryImpl(pExprTree->_node.pLeft, pBuf); + exprTreeToBinaryImpl(pExprTree->_node.pRight, pBuf); } +} - tfree(pRes->pRes); - pRes->num = 0; +SBuffer exprTreeToBinary(tExprNode* pExprTree) { + SBuffer buf = {0}; + if (pExprTree == NULL) { + return buf; + } + + int32_t code = tbufBeginWrite(&buf); + if (code != 0) { + return buf; + } + + exprTreeToBinaryImpl(pExprTree, &buf); + return buf; } + +static tExprNode* exprTreeFromBinaryImpl(tExprNode** pExprTree, SBuffer* pBuf) { + tExprNode* pExpr = calloc(1, sizeof(tExprNode)); + tbufReadToBuffer(pBuf, &pExpr->nodeType, sizeof(pExpr->nodeType)); + + if (pExpr->nodeType == TSQL_NODE_VALUE) { + tVariant* pVal = calloc(1, sizeof(tVariant)); + + tbufReadToBuffer(pBuf, &pVal->nType, sizeof(pVal->nType)); + if (pVal->nType == TSDB_DATA_TYPE_BINARY) { + tbufReadToBuffer(pBuf, &pVal->nLen, sizeof(pVal->nLen)); + pVal->pz = calloc(1, pVal->nLen + 1); + tbufReadToBuffer(pBuf, pVal->pz, pVal->nLen); + } else { + tbufReadToBuffer(pBuf, &pVal->pz, sizeof(pVal->i64Key)); + } + + pExpr->pVal = pVal; + } else if (pExpr->nodeType == TSQL_NODE_COL) { + SSchema* pSchema = calloc(1, sizeof(SSchema)); + tbufReadToBuffer(pBuf, &pSchema->colId, sizeof(pSchema->colId)); + tbufReadToBuffer(pBuf, &pSchema->bytes, sizeof(pSchema->bytes)); + tbufReadToBuffer(pBuf, &pSchema->type, sizeof(pSchema->type)); + + tbufReadToString(pBuf, pSchema->name, TSDB_COL_NAME_LEN); + + pExpr->pSchema = pSchema; + } else if (pExpr->nodeType == TSQL_NODE_EXPR) { + tbufReadToBuffer(pBuf, &pExpr->_node.optr, sizeof(pExpr->_node.optr)); + tbufReadToBuffer(pBuf, &pExpr->_node.hasPK, sizeof(pExpr->_node.hasPK)); + + exprTreeFromBinaryImpl(&pExpr->_node.pLeft, pBuf); + exprTreeFromBinaryImpl(&pExpr->_node.pRight, pBuf); + + assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL); + } + + *pExprTree = pExpr; +} + +tExprNode* exprTreeFromBinary(const void* pBuf, size_t size) { + SBuffer rbuf = {0}; + int32_t code = tbufBeginRead(&rbuf, pBuf, size); + + tExprNode* pExprNode = NULL; + exprTreeFromBinaryImpl(&pExprNode, &rbuf); + return pExprNode; +} \ No newline at end of file diff --git a/src/query/src/qparserImpl.c b/src/query/src/qparserImpl.c index e4067d470fc93617f93623aafc7a59138090d7c0..a40b1bb8d21b1e4dcad2036611ea9e1e95d0062a 100644 --- a/src/query/src/qparserImpl.c +++ b/src/query/src/qparserImpl.c @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#include #include "os.h" #include "qsqlparser.h" +#include "taosdef.h" #include "taosmsg.h" #include "tglobalcfg.h" #include "tlog.h" #include "tstoken.h" #include "ttime.h" #include "ttokendef.h" -#include "taosdef.h" #include "tutil.h" +#include "qsqltype.h" +#include "tstrbuild.h" int32_t tSQLParse(SSqlInfo *pSQLInfo, const char *pStr) { void *pParser = ParseAlloc(malloc); @@ -900,3 +901,173 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { memset(&pDBInfo->precision, 0, sizeof(SSQLToken)); } + +static bool isExprLeafNode(tSQLExpr* pExpr) { + return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && + (pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) || + pExpr->nSQLOptr == TK_SET); +} + +static bool isExprParentOfLeafNode(tSQLExpr* pExpr) { + return (pExpr->pLeft != NULL && pExpr->pRight != NULL) && + (isExprLeafNode(pExpr->pLeft) && isExprLeafNode(pExpr->pRight)); +} + +static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, SStringBuilder* pBuilder) { + if (pExpr->nSQLOptr == TK_ID) { // column name +// strncpy(*str, pExpr->colInfo.z, pExpr->colInfo.n); +// *str += pExpr->colInfo.n; + + } else if (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_STRING) { // value +// *str += tVariantToString(&pExpr->val, *str); +// taosStringBuilderAppendStringLen() + } else if (pExpr->nSQLOptr >= TK_COUNT && pExpr->nSQLOptr <= TK_AVG_IRATE) { + taosStringBuilderAppendStringLen(pBuilder, pExpr->operand.z, pExpr->operand.n); + } else { // not supported operation + assert(false); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t optrToString(tSQLExpr* pExpr, char** exprString) { + const char* le = "<="; + const char* ge = ">="; + const char* ne = "<>"; + const char* likeOptr = "LIKE"; + + switch (pExpr->nSQLOptr) { + case TK_LE: { + *(int16_t*)(*exprString) = *(int16_t*)le; + *exprString += 1; + break; + } + case TK_GE: { + *(int16_t*)(*exprString) = *(int16_t*)ge; + *exprString += 1; + break; + } + case TK_NE: { + *(int16_t*)(*exprString) = *(int16_t*)ne; + *exprString += 1; + break; + } + + case TK_LT: + *(*exprString) = '<'; + break; + case TK_GT: + *(*exprString) = '>'; + break; + case TK_EQ: + *(*exprString) = '='; + break; + case TK_PLUS: + *(*exprString) = '+'; + break; + case TK_MINUS: + *(*exprString) = '-'; + break; + case TK_STAR: + *(*exprString) = '*'; + break; + case TK_DIVIDE: + *(*exprString) = '/'; + break; + case TK_REM: + *(*exprString) = '%'; + break; + case TK_LIKE: { + int32_t len = sprintf(*exprString, " %s ", likeOptr); + *exprString += (len - 1); + break; + } + default: + return TSDB_CODE_INVALID_SQL; + } + + *exprString += 1; + return TSDB_CODE_SUCCESS; +} + +static int32_t tSQLExprLeafToBinary(tSQLExpr* pExpr, SStringBuilder* pBuilder) { + if (!isExprParentOfLeafNode(pExpr)) { + return TSDB_CODE_INVALID_SQL; + } + + tSQLExpr* pLeft = pExpr->pLeft; + tSQLExpr* pRight = pExpr->pRight; + +// if (addParentheses) { +// *(*output) = '('; +// *output += 1; +// } + + tSQLExprNodeToString(pLeft, pBuilder); + tSQLExprNodeToString(pRight, pBuilder); + + if (optrToString(pExpr, pBuilder) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_INVALID_SQL; + } + +// if (addParentheses) { +// *(*output) = ')'; +// *output += 1; +// } + + return TSDB_CODE_SUCCESS; +} + +static void relToString(tSQLExpr* pExpr, SStringBuilder* pBuilder) { + assert(pExpr->nSQLOptr == TK_AND || pExpr->nSQLOptr == TK_OR); + + const char* or = "OR"; + const char*and = "AND"; + + // if (pQueryInfo->tagCond.relType == TSQL_STABLE_QTYPE_COND) { +// if (pExpr->nSQLOptr == TK_AND) { +// strcpy(*str, and); +// *str += strlen(and); +// } else { +// strcpy(*str, or); +// *str += strlen(or); +// } +} + +static int32_t doSQLExprToBinary(tSQLExpr* pExpr, SStringBuilder* pBuilder) { + if (pExpr == NULL) { + return TSDB_CODE_SUCCESS; + } + + if (!isExprParentOfLeafNode(pExpr)) { +// *(*str) = '('; +// *str += 1; + + int32_t ret = doSQLExprToBinary(pExpr->pLeft, pBuilder); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + ret = doSQLExprToBinary(pExpr->pRight, pBuilder); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + relToString(pExpr, pBuilder); + +// *(*str) = ')'; +// *str += 1; + + return ret; + } + + return tSQLExprLeafToBinary(pExpr, pBuilder); +} + +// post order seralize to binary data +int32_t tSQLExprToBinary(tSQLExpr* pExpr, SStringBuilder* pBuilder) { + assert(pExpr != NULL && pBuilder != NULL); + + + +} \ No newline at end of file diff --git a/src/query/src/qtokenizer.c b/src/query/src/qtokenizer.c index 61d2e59c87fa2dc136d8e51869c10fd6028a9127..51b196a9da2c9282e2fa65ded7088cdb0bdf9ca0 100644 --- a/src/query/src/qtokenizer.c +++ b/src/query/src/qtokenizer.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ +#include "os.h" + #include "hash.h" #include "hashfunc.h" -#include "os.h" -#include "shash.h" #include "taosdef.h" #include "tstoken.h" #include "ttokendef.h" diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 83cbae42664407d1245adfc9740fa31be5af1de0..932c04337745fb4e36cc2b04a4b6829703f4e6ed 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -2603,13 +2603,12 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - int64_t cnt = 0; + dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; - while (tsdbNextDataBlock(pQueryHandle)) { // check if query is killed or not set the status of query to pass the status check if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { @@ -3595,8 +3594,8 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->window.ekey = ekey; STimeWindow win = {.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; - tsdbResetQuery(pRuntimeEnv->pQueryHandle, &win, current, pQuery->order.order); - tsdbNextDataBlock(pRuntimeEnv->pQueryHandle); +// tsdbResetQuery(pRuntimeEnv->pQueryHandle, &win, current, pQuery->order.order); +// tsdbNextDataBlock(pRuntimeEnv->pQueryHandle); } void doFinalizeResult(SQueryRuntimeEnv *pRuntimeEnv) { @@ -5149,7 +5148,7 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { int64_t st = taosGetTimestampUs(); // group by normal column, sliding window query, interval query are handled by interval query processor - if (pQuery->intervalTime != 0 || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) + if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) tableIntervalProcessor(pQInfo); } else { if (isFixedOutputQuery(pQuery)) { @@ -5461,7 +5460,7 @@ static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMs SSqlBinaryExprInfo *pBinaryExprInfo = &pExpr->binExprInfo; SColumnInfo * pColMsg = pQueryMsg->colList; #if 0 - tSQLBinaryExpr* pBinExpr = NULL; + tExprNode* pBinExpr = NULL; SSchema* pSchema = toSchema(pQueryMsg, pColMsg, pQueryMsg->numOfCols); dTrace("qmsg:%p create binary expr from string:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz); @@ -5962,7 +5961,7 @@ static void freeQInfo(SQInfo *pQInfo) { if (pBinExprInfo->numOfCols > 0) { tfree(pBinExprInfo->pReqColumns); - tSQLBinaryExprDestroy(&pBinExprInfo->pBinExpr, NULL); + tExprTreeDestroy(&pBinExprInfo->pBinExpr, NULL); } } diff --git a/src/query/src/queryUtil.c b/src/query/src/queryUtil.c index c970363c01fc148a58da8dd78cf0a43c255297cc..17410b2868a74f778cc566a4e5b69dc2d6a4f408 100644 --- a/src/query/src/queryUtil.c +++ b/src/query/src/queryUtil.c @@ -21,10 +21,10 @@ #include "ttime.h" #include "qinterpolation.h" -//#include "tscJoinProcess.h" #include "ttime.h" #include "queryExecutor.h" +#include "queryUtil.h" int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size, int32_t threshold, int16_t type) { diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ac10cd2429ea5a7634fe226d4bc9f80f17e44e28 --- /dev/null +++ b/src/query/tests/astTest.cpp @@ -0,0 +1,630 @@ +#include +#include +#include +#include +#include + +#include "qast.h" +#include "taosmsg.h" +#include "tsdb.h" +#include "tskiplist.h" + +typedef struct ResultObj { + int32_t numOfResult; + char * resultName[64]; +} ResultObj; + +static void initSchema(SSchema *pSchema, int32_t numOfCols); + +static void initSchema_binary(SSchema *schema, int32_t numOfCols); + +static tSkipList *createSkipList(SSchema *pSchema, int32_t numOfTags); +static tSkipList *createSkipList_binary(SSchema *pSchema, int32_t numOfTags); + +static void testQueryStr(SSchema *schema, int32_t numOfCols, char *sql, tSkipList *pSkipList, ResultObj *expectedVal); + +static void dropMeter(tSkipList *pSkipList); + +static void Right2LeftTest(SSchema *schema, int32_t numOfCols, tSkipList *pSkipList); + +static void Left2RightTest(SSchema *schema, int32_t numOfCols, tSkipList *pSkipList); + +static void IllegalExprTest(SSchema *schema, int32_t numOfCols, tSkipList *pSkipList); + +static void Left2RightTest_binary(SSchema *schema, int32_t numOfCols, tSkipList *pSkipList); +static void Right2LeftTest_binary(SSchema *schema, int32_t numOfCols, tSkipList *pSkipList); + +void setValue(ResultObj *pResult, int32_t num, char **val) { + pResult->numOfResult = num; + for (int32_t i = 0; i < num; ++i) { + pResult->resultName[i] = val[i]; + } +} + +static void initSchema_binary(SSchema *schema, int32_t numOfCols) { + schema[0].type = TSDB_DATA_TYPE_BINARY; + schema[0].bytes = 8; + strcpy(schema[0].name, "a"); + + schema[1].type = TSDB_DATA_TYPE_DOUBLE; + schema[1].bytes = 8; + strcpy(schema[1].name, "b"); + + schema[2].type = TSDB_DATA_TYPE_INT; + schema[2].bytes = 20; + strcpy(schema[2].name, "c"); + + schema[3].type = TSDB_DATA_TYPE_BIGINT; + schema[3].bytes = 8; + strcpy(schema[3].name, "d"); + + schema[4].type = TSDB_DATA_TYPE_SMALLINT; + schema[4].bytes = 2; + strcpy(schema[4].name, "e"); + + schema[5].type = TSDB_DATA_TYPE_TINYINT; + schema[5].bytes = 1; + strcpy(schema[5].name, "f"); + + schema[6].type = TSDB_DATA_TYPE_FLOAT; + schema[6].bytes = 4; + strcpy(schema[6].name, "g"); + + schema[7].type = TSDB_DATA_TYPE_BOOL; + schema[7].bytes = 1; + strcpy(schema[7].name, "h"); +} + +static void initSchema(SSchema *schema, int32_t numOfCols) { + schema[0].type = TSDB_DATA_TYPE_INT; + schema[0].bytes = 8; + strcpy(schema[0].name, "a"); + + schema[1].type = TSDB_DATA_TYPE_DOUBLE; + schema[1].bytes = 8; + strcpy(schema[1].name, "b"); + + schema[2].type = TSDB_DATA_TYPE_BINARY; + schema[2].bytes = 20; + strcpy(schema[2].name, "c"); + + schema[3].type = TSDB_DATA_TYPE_BIGINT; + schema[3].bytes = 8; + strcpy(schema[3].name, "d"); + + schema[4].type = TSDB_DATA_TYPE_SMALLINT; + schema[4].bytes = 2; + strcpy(schema[4].name, "e"); + + schema[5].type = TSDB_DATA_TYPE_TINYINT; + schema[5].bytes = 1; + strcpy(schema[5].name, "f"); + + schema[6].type = TSDB_DATA_TYPE_FLOAT; + schema[6].bytes = 4; + strcpy(schema[6].name, "g"); + + schema[7].type = TSDB_DATA_TYPE_BOOL; + schema[7].bytes = 1; + strcpy(schema[7].name, "h"); +} + +// static void addOneNode(SSchema *pSchema, int32_t tagsLen, tSkipList *pSkipList, +// char *meterId, int32_t a, double b, char *c, int64_t d, int16_t e, int8_t f, float g, +// bool h, int32_t numOfTags) { +// STabObj *pMeter = calloc(1, sizeof(STabObj)); +// pMeter->numOfTags = numOfTags; +// pMeter->pTagData = calloc(1, tagsLen + TSDB_METER_ID_LEN); +// strcpy(pMeter->meterId, meterId); +// +// char *tags = pMeter->pTagData + TSDB_METER_ID_LEN; +// int32_t offset = 0; +// +// *(int32_t *) tags = a; +// +// offset += pSchema[0].bytes; +// *(double *) (tags + offset) = b; +// +// offset += pSchema[1].bytes; +// memcpy(tags + offset, c, 3); +// +// offset += pSchema[2].bytes; +// *(int64_t *) (tags + offset) = d; +// +// offset += pSchema[3].bytes; +// *(int16_t *) (tags + offset) = e; +// +// offset += pSchema[4].bytes; +// *(int8_t *) (tags + offset) = f; +// +// offset += pSchema[5].bytes; +// *(float *) (tags + offset) = g; +// +// offset += pSchema[6].bytes; +// *(int8_t *) (tags + offset) = h ? 1 : 0; +// +// tSkipListKey pKey = tSkipListCreateKey(pSchema[0].type, tags, pSchema[0].bytes); +// tSkipListPut(pSkipList, pMeter, &pKey, 1); +//} +// +// static void addOneNode_binary(SSchema *pSchema, int32_t tagsLen, tSkipList *pSkipList, +// char *meterId, int32_t a, double b, char *c, int64_t d, int16_t e, int8_t f, float g, +// bool h, int32_t numOfTags) { +// STabObj *pMeter = calloc(1, sizeof(STabObj)); +// pMeter->numOfTags = numOfTags; +// pMeter->pTagData = calloc(1, tagsLen + TSDB_METER_ID_LEN); +// strcpy(pMeter->meterId, meterId); +// +// char *tags = pMeter->pTagData + TSDB_METER_ID_LEN; +// int32_t offset = 0; +// memcpy(tags, c, pSchema[0].bytes); +// +// offset += pSchema[0].bytes; +// *(double *) (tags + offset) = b; +// +// offset += pSchema[1].bytes; +// *(int32_t *) (tags + offset) = a; +// +// offset += pSchema[2].bytes; +// *(int64_t *) (tags + offset) = d; +// +// offset += pSchema[3].bytes; +// *(int16_t *) (tags + offset) = e; +// +// offset += pSchema[4].bytes; +// *(int8_t *) (tags + offset) = f; +// +// offset += pSchema[5].bytes; +// *(float *) (tags + offset) = g; +// +// offset += pSchema[6].bytes; +// *(int8_t *) (tags + offset) = h ? 1 : 0; +// +// tSkipListKey pKey = tSkipListCreateKey(pSchema[0].type, tags, pSchema[0].bytes); +// tSkipListPut(pSkipList, pMeter, &pKey, 1); +// tSkipListDestroyKey(&pKey); +//} + +// static void dropMeter(tSkipList *pSkipList) { +// tSkipListNode **pRes = NULL; +// int32_t num = tSkipListIterateList(pSkipList, &pRes, NULL, NULL); +// for (int32_t i = 0; i < num; ++i) { +// tSkipListNode *pNode = pRes[i]; +// STabObj *pMeter = (STabObj *) pNode->pData; +// free(pMeter->pTagData); +// free(pMeter); +// pNode->pData = NULL; +// } +// free(pRes); +//} + +// static tSkipList *createSkipList(SSchema *pSchema, int32_t numOfTags) { +// int32_t tagsLen = 0; +// for (int32_t i = 0; i < numOfTags; ++i) { +// tagsLen += pSchema[i].bytes; +// } +// +// tSkipList *pSkipList = tSkipListCreate(10, pSchema[0].type, 4); +// +// addOneNode(pSchema, tagsLen, pSkipList, "tm0\0", 0, 10.5, "abc", 1000, -10000, -20, 1.0, true, 8); +// addOneNode(pSchema, tagsLen, pSkipList, "tm1\0", 1, 20.5, "def", 1100, -10500, -30, 2.0, false, 8); +// addOneNode(pSchema, tagsLen, pSkipList, "tm2\0", 2, 30.5, "ghi", 1200, -11000, -40, 3.0, true, 8); +// addOneNode(pSchema, tagsLen, pSkipList, "tm3\0", 3, 40.5, "jkl", 1300, -11500, -50, 4.0, false, 8); +// addOneNode(pSchema, tagsLen, pSkipList, "tm4\0", 4, 50.5, "mno", 1400, -12000, -60, 5.0, true, 8); +// addOneNode(pSchema, tagsLen, pSkipList, "tm5\0", 5, 60.5, "pqr", 1500, -12500, -70, 6.0, false, 8); +// addOneNode(pSchema, tagsLen, pSkipList, "tm6\0", 6, 70.5, "stu", 1600, -13000, -80, 7.0, true, 8); +// +// return pSkipList; +//} +// +// static tSkipList *createSkipList_binary(SSchema *pSchema, int32_t numOfTags) { +// int32_t tagsLen = 0; +// for (int32_t i = 0; i < numOfTags; ++i) { +// tagsLen += pSchema[i].bytes; +// } +// +// tSkipList *pSkipList = tSkipListCreate(10, pSchema[0].type, 4); +// +// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm0\0", 0, 10.5, "abc", 1000, -10000, -20, 1.0, true, 8); +// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm1\0", 1, 20.5, "def", 1100, -10500, -30, 2.0, false, 8); +// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm2\0", 2, 30.5, "ghi", 1200, -11000, -40, 3.0, true, 8); +// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm3\0", 3, 40.5, "jkl", 1300, -11500, -50, 4.0, false, 8); +// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm4\0", 4, 50.5, "mno", 1400, -12000, -60, 5.0, true, 8); +// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm5\0", 5, 60.5, "pqr", 1500, -12500, -70, 6.0, false, 8); +// addOneNode_binary(pSchema, tagsLen, pSkipList, "tm6\0", 6, 70.5, "stu", 1600, -13000, -80, 7.0, true, 8); +// +// return pSkipList; +//} + +static void testQueryStr(SSchema *schema, int32_t numOfCols, char *sql, tSkipList *pSkipList, ResultObj *pResult) { + tExprNode *pExpr = NULL; + tSQLBinaryExprFromString(&pExpr, schema, numOfCols, sql, strlen(sql)); + + char str[512] = {0}; + int32_t len = 0; + if (pExpr == NULL) { + printf("-----error in parse syntax:%s\n\n", sql); + assert(pResult == NULL); + return; + } + + tSQLBinaryExprToString(pExpr, str, &len); + printf("expr is: %s\n", str); + + SArray *result = NULL; + // tSQLBinaryExprTraverse(pExpr, pSkipList, result, tSkipListNodeFilterCallback, &result); + // printf("the result is:%lld\n", result.num); + // + // bool findResult = false; + // for (int32_t i = 0; i < result.num; ++i) { + // STabObj *pm = (STabObj *)result.pRes[i]; + // printf("meterid:%s,\t", pm->meterId); + // + // for (int32_t j = 0; j < pResult->numOfResult; ++j) { + // if (strcmp(pm->meterId, pResult->resultName[j]) == 0) { + // findResult = true; + // break; + // } + // } + // assert(findResult == true); + // findResult = false; + // } + + printf("\n\n"); + tExprTreeDestroy(&pExpr, NULL); +} + +static void Left2RightTest(SSchema *schema, int32_t numOfCols, tSkipList *pSkipList) { + char str[256] = {0}; + + char *t0[1] = {"tm0"}; + char *t1[1] = {"tm1"}; + char *sql = NULL; + + ResultObj res = {1, {"tm1"}}; + testQueryStr(schema, numOfCols, "a=1", pSkipList, &res); + + char *tt[1] = {"tm6"}; + setValue(&res, 1, tt); + testQueryStr(schema, numOfCols, "a>=6", pSkipList, &res); + + setValue(&res, 1, t0); + testQueryStr(schema, numOfCols, "b<=10.6", pSkipList, &res); + + strcpy(str, "c<>'pqr'"); + char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"}; + setValue(&res, 6, t2); + testQueryStr(schema, numOfCols, str, pSkipList, &res); + + strcpy(str, "c='abc'"); + setValue(&res, 1, t0); + testQueryStr(schema, numOfCols, str, pSkipList, &res); + + char *t3[6] = {"tm1", "tm2", "tm3", "tm4", "tm5", "tm6"}; + setValue(&res, 6, t3); + testQueryStr(schema, numOfCols, "d>1050", pSkipList, &res); + + char *t4[3] = {"tm4", "tm5", "tm6"}; + setValue(&res, 3, t4); + testQueryStr(schema, numOfCols, "g>4.5980765", pSkipList, &res); + + char *t5[4] = {"tm0", "tm2", "tm4", "tm6"}; + setValue(&res, 4, t5); + testQueryStr(schema, numOfCols, "h=true", pSkipList, &res); + + char *t6[3] = {"tm1", "tm3", "tm5"}; + setValue(&res, 3, t6); + testQueryStr(schema, numOfCols, "h=0", pSkipList, &res); + + sql = "(((b<40)))\0"; + char *t7[3] = {"tm0", "tm1", "tm2"}; + setValue(&res, 3, t7); + testQueryStr(schema, numOfCols, sql, pSkipList, &res); + + sql = "((a=1) or (a=10)) or ((b=12))"; + setValue(&res, 1, t1); + testQueryStr(schema, numOfCols, sql, pSkipList, &res); + + sql = "((((((a>0 and a<2))) or a=6) or a=3) or (b=50.5)) and h=0"; + char *t8[2] = {"tm1", "tm3"}; + setValue(&res, 2, t8); + testQueryStr(schema, numOfCols, sql, pSkipList, &res); + + char *tf[1] = {"tm6"}; + setValue(&res, 1, tf); + testQueryStr(schema, numOfCols, "e = -13000", pSkipList, &res); + + char *ft[5] = {"tm0", "tm1", "tm2", "tm3", "tm4"}; + setValue(&res, 5, ft); + testQueryStr(schema, numOfCols, "f > -65", pSkipList, &res); +} + +void Right2LeftTest(SSchema *schema, int32_t numOfCols, tSkipList *pSkipList) { + ResultObj res = {1, {"tm1"}}; + testQueryStr(schema, numOfCols, "((1=a))", pSkipList, &res); + + char *t9[2] = {"tm0", "tm1"}; + setValue(&res, 2, t9); + testQueryStr(schema, numOfCols, "1>=a", pSkipList, &res); + + char *t0[1] = {"tm0"}; + setValue(&res, 1, t0); + testQueryStr(schema, numOfCols, "10.6>=b", pSkipList, &res); + + char *t10[3] = {"tm1", "tm3", "tm5"}; + setValue(&res, 3, t10); + testQueryStr(schema, numOfCols, "0=h", pSkipList, &res); +} + +static void IllegalExprTest(SSchema *schema, int32_t numOfCols, tSkipList *pSkipList) { + testQueryStr(schema, numOfCols, "h=", pSkipList, NULL); + testQueryStr(schema, numOfCols, "h<", pSkipList, NULL); + testQueryStr(schema, numOfCols, "a=1 and ", pSkipList, NULL); + testQueryStr(schema, numOfCols, "and or", pSkipList, NULL); + testQueryStr(schema, numOfCols, "and a = 1 or", pSkipList, NULL); + testQueryStr(schema, numOfCols, "(())", pSkipList, NULL); + testQueryStr(schema, numOfCols, "(", pSkipList, NULL); + testQueryStr(schema, numOfCols, "(a", pSkipList, NULL); + testQueryStr(schema, numOfCols, "(a)", pSkipList, NULL); + testQueryStr(schema, numOfCols, "())", pSkipList, NULL); + testQueryStr(schema, numOfCols, "a===1", pSkipList, NULL); + testQueryStr(schema, numOfCols, "a=1 and ", pSkipList, NULL); +} + +static void Left2RightTest_binary(SSchema *schema, int32_t numOfCols, tSkipList *pSkipList) { + char str[256] = {0}; + char *sql = NULL; + + char *t0[1] = {"tm0"}; + char *t1[1] = {"tm1"}; + + ResultObj res = {1, {"tm0"}}; + strcpy(str, "a='abc'"); + testQueryStr(schema, numOfCols, str, pSkipList, &res); + + char *tt[1] = {"tm6"}; + setValue(&res, 1, tt); + testQueryStr(schema, numOfCols, "c>=6", pSkipList, &res); + + setValue(&res, 1, t0); + testQueryStr(schema, numOfCols, "b<=10.6", pSkipList, &res); + + strcpy(str, "a<>'pqr'"); + char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"}; + setValue(&res, 6, t2); + testQueryStr(schema, numOfCols, str, pSkipList, &res); + + strcpy(str, "a='abc'"); + setValue(&res, 1, t0); + testQueryStr(schema, numOfCols, str, pSkipList, &res); + + char *t3[6] = {"tm1", "tm2", "tm3", "tm4", "tm5", "tm6"}; + setValue(&res, 6, t3); + testQueryStr(schema, numOfCols, "d>1050", pSkipList, &res); + + char *t4[3] = {"tm4", "tm5", "tm6"}; + setValue(&res, 3, t4); + testQueryStr(schema, numOfCols, "g>4.5980765", pSkipList, &res); + + char *t5[4] = {"tm0", "tm2", "tm4", "tm6"}; + setValue(&res, 4, t5); + testQueryStr(schema, numOfCols, "h=true", pSkipList, &res); + + char *t6[3] = {"tm1", "tm3", "tm5"}; + setValue(&res, 3, t6); + testQueryStr(schema, numOfCols, "h=0", pSkipList, &res); + + sql = "(((b<40)))\0"; + char *t7[3] = {"tm0", "tm1", "tm2"}; + setValue(&res, 3, t7); + testQueryStr(schema, numOfCols, sql, pSkipList, &res); + + sql = "((c=1) or (c=10)) or ((b=12))\0"; + setValue(&res, 1, t1); + testQueryStr(schema, numOfCols, sql, pSkipList, &res); + + sql = "((((((c>0 and c<2))) or c=6) or c=3) or (b=50.5)) and h=false\0"; + char *t8[2] = {"tm1", "tm3"}; + setValue(&res, 2, t8); + testQueryStr(schema, numOfCols, sql, pSkipList, &res); +} + +static void Right2LeftTest_binary(SSchema *schema, int32_t numOfCols, tSkipList *pSkipList) { + char str[256] = {0}; + char *sql = NULL; + + char *t0[1] = {"tm0"}; + char *t1[1] = {"tm1"}; + + ResultObj res = {1, {"tm0"}}; + strcpy(str, "'abc'=a"); + testQueryStr(schema, numOfCols, str, pSkipList, &res); + + char *tt[1] = {"tm6"}; + setValue(&res, 1, tt); + testQueryStr(schema, numOfCols, "6<=c", pSkipList, &res); + + setValue(&res, 1, t0); + testQueryStr(schema, numOfCols, "10.6>=b", pSkipList, &res); + + strcpy(str, "'pqr'<>a"); + char *t2[6] = {"tm0", "tm1", "tm2", "tm3", "tm4", "tm6"}; + setValue(&res, 6, t2); + testQueryStr(schema, numOfCols, str, pSkipList, &res); +} + +namespace { +// two level expression tree +tExprNode *createExpr1() { + auto *pLeft = (tExprNode*) calloc(1, sizeof(tExprNode)); + pLeft->nodeType = TSQL_NODE_COL; + pLeft->pSchema = (SSchema*) calloc(1, sizeof(SSchema)); + + strcpy(pLeft->pSchema->name, "col_a"); + pLeft->pSchema->type = TSDB_DATA_TYPE_INT; + pLeft->pSchema->bytes = sizeof(int32_t); + pLeft->pSchema->colId = 1; + + auto *pRight = (tExprNode*) calloc(1, sizeof(tExprNode)); + pRight->nodeType = TSQL_NODE_VALUE; + pRight->pVal = (tVariant*) calloc(1, sizeof(tVariant)); + + pRight->pVal->nType = TSDB_DATA_TYPE_INT; + pRight->pVal->i64Key = 12; + + auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode)); + pRoot->nodeType = TSQL_NODE_EXPR; + + pRoot->_node.optr = TSDB_RELATION_EQUAL; + pRoot->_node.pLeft = pLeft; + pRoot->_node.pRight = pRight; + pRoot->_node.hasPK = true; + + return pRoot; +} + +// thress level expression tree +tExprNode* createExpr2() { + auto *pLeft2 = (tExprNode*) calloc(1, sizeof(tExprNode)); + pLeft2->nodeType = TSQL_NODE_COL; + pLeft2->pSchema = (SSchema*) calloc(1, sizeof(SSchema)); + + strcpy(pLeft2->pSchema->name, "col_a"); + pLeft2->pSchema->type = TSDB_DATA_TYPE_BINARY; + pLeft2->pSchema->bytes = 20; + pLeft2->pSchema->colId = 1; + + auto *pRight2 = (tExprNode*) calloc(1, sizeof(tExprNode)); + pRight2->nodeType = TSQL_NODE_VALUE; + pRight2->pVal = (tVariant*) calloc(1, sizeof(tVariant)); + + pRight2->pVal->nType = TSDB_DATA_TYPE_BINARY; + const char* v = "hello world!"; + pRight2->pVal->pz = strdup(v); + pRight2->pVal->nLen = strlen(v); + + auto *p1 = (tExprNode*) calloc(1, sizeof(tExprNode)); + p1->nodeType = TSQL_NODE_EXPR; + + p1->_node.optr = TSDB_RELATION_LIKE; + p1->_node.pLeft = pLeft2; + p1->_node.pRight = pRight2; + p1->_node.hasPK = false; + + auto *pLeft1 = (tExprNode*) calloc(1, sizeof(tExprNode)); + pLeft1->nodeType = TSQL_NODE_COL; + pLeft1->pSchema = (SSchema*) calloc(1, sizeof(SSchema)); + + strcpy(pLeft1->pSchema->name, "col_b"); + pLeft1->pSchema->type = TSDB_DATA_TYPE_DOUBLE; + pLeft1->pSchema->bytes = 8; + pLeft1->pSchema->colId = 99; + + auto *pRight1 = (tExprNode*) calloc(1, sizeof(tExprNode)); + pRight1->nodeType = TSQL_NODE_VALUE; + pRight1->pVal = (tVariant*) calloc(1, sizeof(tVariant)); + + pRight1->pVal->nType = TSDB_DATA_TYPE_DOUBLE; + pRight1->pVal->dKey = 91.99; + + auto *p2 = (tExprNode*) calloc(1, sizeof(tExprNode)); + p2->nodeType = TSQL_NODE_EXPR; + + p2->_node.optr = TSDB_RELATION_LARGE_EQUAL; + p2->_node.pLeft = pLeft1; + p2->_node.pRight = pRight1; + p2->_node.hasPK = false; + + auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode)); + pRoot->nodeType = TSQL_NODE_EXPR; + + pRoot->_node.optr = TSDB_RELATION_OR; + pRoot->_node.pLeft = p1; + pRoot->_node.pRight = p2; + pRoot->_node.hasPK = true; + return pRoot; +} + +void exprSerializeTest1() { + tExprNode* p1 = createExpr1(); + SBuffer buf = exprTreeToBinary(p1); + + size_t size = tbufTell(&buf); + ASSERT_TRUE(size > 0); + char* b = tbufGetData(&buf, false); + + tExprNode* p2 = exprTreeFromBinary(b, size); + ASSERT_EQ(p1->nodeType, p2->nodeType); + + ASSERT_EQ(p2->_node.optr, p1->_node.optr); + ASSERT_EQ(p2->_node.pLeft->nodeType, p1->_node.pLeft->nodeType); + ASSERT_EQ(p2->_node.pRight->nodeType, p1->_node.pRight->nodeType); + + SSchema* s1 = p1->_node.pLeft->pSchema; + SSchema* s2 = p2->_node.pLeft->pSchema; + + ASSERT_EQ(s2->colId, s1->colId); + ASSERT_EQ(s2->type, s1->type); + ASSERT_EQ(s2->bytes, s1->bytes); + ASSERT_STRCASEEQ(s2->name, s1->name); + + tVariant* v1 = p1->_node.pRight->pVal; + tVariant* v2 = p2->_node.pRight->pVal; + + ASSERT_EQ(v1->nType, v2->nType); + ASSERT_EQ(v1->i64Key, v2->i64Key); + ASSERT_EQ(p1->_node.hasPK, p2->_node.hasPK); + + tExprTreeDestroy(&p1, nullptr); + tExprTreeDestroy(&p2, nullptr); + + tbufClose(&buf, false); +} + +void exprSerializeTest2() { + tExprNode* p1 = createExpr2(); + SBuffer buf = exprTreeToBinary(p1); + + size_t size = tbufTell(&buf); + ASSERT_TRUE(size > 0); + char* b = tbufGetData(&buf, false); + + tExprNode* p2 = exprTreeFromBinary(b, size); + ASSERT_EQ(p1->nodeType, p2->nodeType); + + ASSERT_EQ(p2->_node.optr, p1->_node.optr); + ASSERT_EQ(p2->_node.pLeft->nodeType, p1->_node.pLeft->nodeType); + ASSERT_EQ(p2->_node.pRight->nodeType, p1->_node.pRight->nodeType); + + tExprNode* c1Left = p1->_node.pLeft; + tExprNode* c2Left = p2->_node.pLeft; + + ASSERT_EQ(c1Left->nodeType, c2Left->nodeType); + + ASSERT_EQ(c2Left->nodeType, TSQL_NODE_EXPR); + ASSERT_EQ(c2Left->_node.optr, TSDB_RELATION_LIKE); + + ASSERT_STRCASEEQ(c2Left->_node.pLeft->pSchema->name, "col_a"); + ASSERT_EQ(c2Left->_node.pRight->nodeType, TSQL_NODE_VALUE); + + ASSERT_STRCASEEQ(c2Left->_node.pRight->pVal->pz, "hello world!"); + + tExprNode* c1Right = p1->_node.pRight; + tExprNode* c2Right = p2->_node.pRight; + + ASSERT_EQ(c1Right->nodeType, c2Right->nodeType); + ASSERT_EQ(c2Right->nodeType, TSQL_NODE_EXPR); + ASSERT_EQ(c2Right->_node.optr, TSDB_RELATION_LARGE_EQUAL); + ASSERT_EQ(c2Right->_node.pRight->pVal->dKey, 91.99); + + ASSERT_EQ(p2->_node.hasPK, true); + + tExprTreeDestroy(&p1, nullptr); + tExprTreeDestroy(&p2, nullptr); + + tbufClose(&buf, false); +} +} // namespace +TEST(testCase, astTest) { + exprSerializeTest2(); +} \ No newline at end of file diff --git a/src/util/inc/tbuffer.h b/src/util/inc/tbuffer.h index 9dc6d97eb6acce4077cc2e7b93c05a49839e68f7..d52031fc6a63252a393189d2c4bf9aa2ff622ded 100644 --- a/src/util/inc/tbuffer.h +++ b/src/util/inc/tbuffer.h @@ -13,14 +13,15 @@ * along with this program. If not, see . */ -#include -#include -#include -#include - #ifndef TDENGINE_TBUFFER_H #define TDENGINE_TBUFFER_H +#include "setjmp.h" +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif /* SBuffer can be used to read or write a buffer, but cannot be used for both @@ -80,37 +81,33 @@ int main(int argc, char** argv) { */ typedef struct { jmp_buf jb; - char* data; - size_t pos; - size_t size; + char* data; + size_t pos; + size_t size; } SBuffer; - // common functions can be used in both read & write #define tbufThrowError(buf, code) longjmp((buf)->jb, (code)) size_t tbufTell(SBuffer* buf); size_t tbufSeekTo(SBuffer* buf, size_t pos); size_t tbufSkip(SBuffer* buf, size_t size); -void tbufClose(SBuffer* buf, bool keepData); - +void tbufClose(SBuffer* buf, bool keepData); // basic read functions -#define tbufBeginRead(buf, data, len) (((buf)->data = (char*)data), ((buf)->pos = 0), ((buf)->size = ((data) == NULL) ? 0 : (len)), setjmp((buf)->jb)) -char* tbufRead(SBuffer* buf, size_t size); -void tbufReadToBuffer(SBuffer* buf, void* dst, size_t size); +#define tbufBeginRead(buf, _data, len) ((buf)->data = (char*)(_data), ((buf)->pos = 0), ((buf)->size = ((_data) == NULL) ? 0 : (len)), setjmp((buf)->jb)) +char* tbufRead(SBuffer* buf, size_t size); +void tbufReadToBuffer(SBuffer* buf, void* dst, size_t size); const char* tbufReadString(SBuffer* buf, size_t* len); -size_t tbufReadToString(SBuffer* buf, char* dst, size_t size); - +size_t tbufReadToString(SBuffer* buf, char* dst, size_t size); // basic write functions #define tbufBeginWrite(buf) ((buf)->data = NULL, ((buf)->pos = 0), ((buf)->size = 0), setjmp((buf)->jb)) -void tbufEnsureCapacity(SBuffer* buf, size_t size); +void tbufEnsureCapacity(SBuffer* buf, size_t size); char* tbufGetData(SBuffer* buf, bool takeOver); -void tbufWrite(SBuffer* buf, const void* data, size_t size); -void tbufWriteAt(SBuffer* buf, size_t pos, const void* data, size_t size); -void tbufWriteStringLen(SBuffer* buf, const char* str, size_t len); -void tbufWriteString(SBuffer* buf, const char* str); - +void tbufWrite(SBuffer* buf, const void* data, size_t size); +void tbufWriteAt(SBuffer* buf, size_t pos, const void* data, size_t size); +void tbufWriteStringLen(SBuffer* buf, const char* str, size_t len); +void tbufWriteString(SBuffer* buf, const char* str); // read & write function for primitive types #ifndef TBUFFER_DEFINE_FUNCTION @@ -120,17 +117,21 @@ void tbufWriteString(SBuffer* buf, const char* str); void tbufWrite##name##At(SBuffer* buf, size_t pos, type data); #endif -TBUFFER_DEFINE_FUNCTION( bool, Bool ) -TBUFFER_DEFINE_FUNCTION( char, Char ) -TBUFFER_DEFINE_FUNCTION( int8_t, Int8 ) -TBUFFER_DEFINE_FUNCTION( uint8_t, Unt8 ) -TBUFFER_DEFINE_FUNCTION( int16_t, Int16 ) -TBUFFER_DEFINE_FUNCTION( uint16_t, Uint16 ) -TBUFFER_DEFINE_FUNCTION( int32_t, Int32 ) -TBUFFER_DEFINE_FUNCTION( uint32_t, Uint32 ) -TBUFFER_DEFINE_FUNCTION( int64_t, Int64 ) -TBUFFER_DEFINE_FUNCTION( uint64_t, Uint64 ) -TBUFFER_DEFINE_FUNCTION( float, Float ) -TBUFFER_DEFINE_FUNCTION( double, Double ) +TBUFFER_DEFINE_FUNCTION(bool, Bool) +TBUFFER_DEFINE_FUNCTION(char, Char) +TBUFFER_DEFINE_FUNCTION(int8_t, Int8) +TBUFFER_DEFINE_FUNCTION(uint8_t, Unt8) +TBUFFER_DEFINE_FUNCTION(int16_t, Int16) +TBUFFER_DEFINE_FUNCTION(uint16_t, Uint16) +TBUFFER_DEFINE_FUNCTION(int32_t, Int32) +TBUFFER_DEFINE_FUNCTION(uint32_t, Uint32) +TBUFFER_DEFINE_FUNCTION(int64_t, Int64) +TBUFFER_DEFINE_FUNCTION(uint64_t, Uint64) +TBUFFER_DEFINE_FUNCTION(float, Float) +TBUFFER_DEFINE_FUNCTION(double, Double) + +#ifdef __cplusplus +} +#endif #endif \ No newline at end of file diff --git a/src/util/inc/tstatus.h b/src/util/inc/tstatus.h index 9a65bb3ac118e27fa632753fc823431d5f619488..223ed1d52dcb8dea8f832fd26f7972333eb9550c 100644 --- a/src/util/inc/tstatus.h +++ b/src/util/inc/tstatus.h @@ -62,6 +62,19 @@ enum _TSDB_VN_DROP_STATUS { TSDB_VN_DROP_STATUS_DROPPING }; +enum _TSDB_MN_STATUS { + TSDB_MN_STATUS_OFFLINE, + TSDB_MN_STATUS_UNSYNCED, + TSDB_MN_STATUS_SYNCING, + TSDB_MN_STATUS_SERVING +}; + +enum _TSDB_MN_ROLE { + TSDB_MN_ROLE_UNDECIDED, + TSDB_MN_ROLE_SLAVE, + TSDB_MN_ROLE_MASTER +}; + enum _TSDB_DN_STATUS { TSDB_DN_STATUS_OFFLINE, TSDB_DN_STATUS_READY @@ -104,6 +117,8 @@ char* taosGetVgroupLbStatusStr(int32_t vglbStatus); char* taosGetVnodeStreamStatusStr(int32_t vnodeStreamStatus); char* taosGetTableStatusStr(int32_t tableStatus); char *taosGetShowTypeStr(int32_t showType); +char *taosGetMnodeStatusStr(int32_t mnodeStatus); +char *taosGetMnodeRoleStr(int32_t mnodeRole); #ifdef __cplusplus } diff --git a/src/util/src/.tqueue.c.swp b/src/util/src/.tqueue.c.swp deleted file mode 100644 index 40a09d5f638ccd61507bd50661e64d34cccce61c..0000000000000000000000000000000000000000 Binary files a/src/util/src/.tqueue.c.swp and /dev/null differ diff --git a/src/util/src/tbuffer.c b/src/util/src/tbuffer.c index ac7d22078d42eea60fde157ea364dee146203a2f..a83d7dddb0d8e987bfbb670f8f3e4b413539dec2 100644 --- a/src/util/src/tbuffer.c +++ b/src/util/src/tbuffer.c @@ -30,7 +30,7 @@ tbufWriteAt(buf, pos, &data, sizeof(data));\ } -#include "../inc/tbuffer.h" +#include "tbuffer.h" //////////////////////////////////////////////////////////////////////////////// @@ -119,13 +119,14 @@ void tbufEnsureCapacity(SBuffer* buf, size_t size) { } char* tbufGetData(SBuffer* buf, bool takeOver) { - char* ret = buf->data; - if (takeOver) { - buf->pos = 0; - buf->size = 0; - buf->data = NULL; - } - return ret; + char* ret = buf->data; + if (takeOver) { + buf->pos = 0; + buf->size = 0; + buf->data = NULL; + } + + return ret; } void tbufEndWrite(SBuffer* buf) { diff --git a/src/util/src/tstatus.c b/src/util/src/tstatus.c index 4cf84304849854be884cad9144141be0d23024c2..e8534de30b82edd1f6182137d85342e7d6f6094b 100644 --- a/src/util/src/tstatus.c +++ b/src/util/src/tstatus.c @@ -136,3 +136,22 @@ char *taosGetShowTypeStr(int32_t showType) { default: return "undefined"; } } + +char *taosGetMnodeStatusStr(int32_t mnodeStatus) { + switch (mnodeStatus) { + case TSDB_MN_STATUS_OFFLINE: return "offline"; + case TSDB_MN_STATUS_UNSYNCED: return "unsynced"; + case TSDB_MN_STATUS_SYNCING: return "syncing"; + case TSDB_MN_STATUS_SERVING: return "serving"; + default: return "undefined"; + } +} + +char *taosGetMnodeRoleStr(int32_t mnodeRole) { + switch (mnodeRole) { + case TSDB_MN_ROLE_UNDECIDED: return "undicided"; + case TSDB_MN_ROLE_SLAVE: return "slave"; + case TSDB_MN_ROLE_MASTER: return "master"; + default: return "undefined"; + } +} diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index ff8da1cdadd130dc28b6544f58620a50429c5643..bd6699eb8484b3ee9b1bbb9cb2415f10f1d7b8f3 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -147,6 +147,11 @@ void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direct } void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { + if (pIter->numOfFGroups == 0) { + assert(pIter->pFileGroup == NULL); + return; + } + int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; void *ptr = taosbsearch(&fid, pIter->base, sizeof(SFileGroup), pIter->numOfFGroups, compFGroupKey, flags); if (ptr == NULL) { diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index d4b6c2e8b6684748b91033cad8dbbbfe8bd8fd17..599d8bd34738d047f2e3331d3a242f08976bf7b9 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -88,8 +88,12 @@ typedef struct STableCheckInfo { bool checkFirstFileBlock; SCompIdx* compIndex; + SCompInfo *pCompInfo; - SCompBlock *pBlock; + SDataCols* pDataCols; + SFileGroup* pFileGroup; + + SFileGroupIter fileIter; SSkipListIterator* iter; } STableCheckInfo; @@ -293,6 +297,8 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond .lastKey = pQueryHandle->window.skey, .tableId = id, .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), //todo this may be failed + .compIndex = calloc(10000, sizeof(SCompIdx)), + .pCompInfo = calloc(1, 1024), }; taosArrayPush(pQueryHandle->pTableCheckInfo, &info); @@ -357,13 +363,18 @@ static int32_t getFileIdFromKey(TSKEY key) { } static int32_t getFileCompInfo(STableCheckInfo* pCheckInfo, SFileGroup* fileGroup) { + // check open file failed + if (fileGroup->files[TSDB_FILE_TYPE_HEAD].fd == FD_INITIALIZER) { + fileGroup->files[TSDB_FILE_TYPE_HEAD].fd = open(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname, O_RDONLY); + } + tsdbLoadCompIdx(fileGroup, pCheckInfo->compIndex, 10000); // todo set dynamic max tables SCompIdx* compIndex = &pCheckInfo->compIndex[pCheckInfo->tableId.tid]; if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file } else { - tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pBlock); + tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); } return TSDB_CODE_SUCCESS; @@ -396,68 +407,120 @@ static int32_t binarySearchForBlockImpl(SCompBlock *pBlock, int32_t numOfBlocks, return midSlot; } -static SDataBlockInfo getTrueBlockInfo(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { - SDataBlockInfo info = {{0}, 0}; +static SDataBlockInfo getTrueDataBlockInfo(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { + SCompBlock *pDiskBlock = &pCheckInfo->pCompInfo->blocks[pHandle->cur.slot]; - SCompBlock *pDiskBlock = &pCheckInfo->pBlock[pHandle->cur.slot]; - - info.window.skey = pDiskBlock->keyFirst; - info.window.ekey = pDiskBlock->keyLast; - info.size = pDiskBlock->numOfPoints; - info.numOfCols = pDiskBlock->numOfCols; + SDataBlockInfo info = { + .window = {.skey = pDiskBlock->keyFirst, .ekey = pDiskBlock->keyLast}, + .numOfCols = pDiskBlock->numOfCols, + .size = pDiskBlock->numOfPoints, + .sid = pCheckInfo->tableId.tid, + .uid = pCheckInfo->tableId.uid, + }; return info; } +SArray *getDefaultLoadColumns(STsdbQueryHandle *pQueryHandle, bool loadTS); +static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCols, SArray *sa); + +static bool doLoadDataFromFileBlock(STsdbQueryHandle *pQueryHandle) { + SQueryFilePos *cur = &pQueryHandle->cur; + + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; + + SCompData* data = calloc(1, sizeof(SCompData)+ sizeof(SCompCol)*pBlock->numOfCols); + + data->numOfCols = pBlock->numOfCols; + data->uid = pCheckInfo->pTableObj->tableId.uid; + + pCheckInfo->pDataCols = tdNewDataCols(1000, 2, 4096); + tdInitDataCols(pCheckInfo->pDataCols, pCheckInfo->pTableObj->schema); + + SFile* pFile = &pCheckInfo->pFileGroup->files[TSDB_FILE_TYPE_DATA]; + if (pFile->fd == FD_INITIALIZER) { + pFile->fd = open(pFile->fname, O_RDONLY); + } + + tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data); +} + +static bool loadQualifiedDataFromFileBlock(STsdbQueryHandle *pQueryHandle) { + SQueryFilePos *cur = &pQueryHandle->cur; + + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; + + SArray *sa = getDefaultLoadColumns(pQueryHandle, true); + if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + + // query ended in current block + if (pQueryHandle->window.ekey < pBlock->keyLast) { + doLoadDataFromFileBlock(pQueryHandle); + filterDataInDataBlock(pQueryHandle, pCheckInfo->pDataCols, sa); + } + } else {// todo desc query + if (pQueryHandle->window.ekey > pBlock->keyFirst) { +// + } + } + + return pQueryHandle->realNumOfRows > 0; +} + bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { SQueryFilePos *cur = &pQueryHandle->cur; if (pQueryHandle->cur.fid >= 0) { - int32_t fileIndex = -1; - /* * 1. ascending order. The last data block of data file * 2. descending order. The first block of file */ - if ((step == QUERY_ASC_FORWARD_STEP && (pQueryHandle->cur.slot == pQueryHandle->numOfBlocks - 1)) || + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + int32_t tid = pCheckInfo->tableId.tid; + + if ((step == QUERY_ASC_FORWARD_STEP && + (pQueryHandle->cur.slot == pCheckInfo->compIndex[tid].numOfSuperBlocks - 1)) || (step == QUERY_DESC_FORWARD_STEP && (pQueryHandle->cur.slot == 0))) { // temporarily keep the position value, in case of no data qualified when move forwards(backwards) SQueryFilePos save = pQueryHandle->cur; - -// fileIndex = getNextDataFileCompInfo(pQueryHandle, &pQueryHandle->cur, &pQueryHandle->vnodeFileInfo, step); - - // first data block in the next file - if (fileIndex >= 0) { - cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; - cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; -// return loadQaulifiedData(pQueryHandle); - } else {// try data in cache - assert(cur->fid == -1); - - if (step == QUERY_ASC_FORWARD_STEP) { -// TSKEY nextTimestamp = -// getQueryStartPositionInCache_rv(pQueryHandle, &pQueryHandle->cur.slot, &pQueryHandle->cur.pos, true); -// if (nextTimestamp < 0) { -// pQueryHandle->cur = save; -// } -// -// return (nextTimestamp > 0); + SFileGroup* fgroup = tsdbGetFileGroupNext(&pCheckInfo->fileIter); + + int32_t fid = -1; + if (fgroup != NULL) { + if ((fid = getFileCompInfo(pCheckInfo, fgroup)) < 0) { + } else { + cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; + cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; + + SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; + SCompData* data = calloc(1, sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols); + + data->numOfCols = pBlock->numOfCols; + data->uid = pCheckInfo->pTableObj->tableId.uid; + + cur->fid = fgroup->fileId; + assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); + + if (pBlock->keyFirst > pQueryHandle->window.ekey) { // done + return false; + } + + loadQualifiedDataFromFileBlock(pQueryHandle); + return true; } - - // no data to check for desc order query, restore the saved position value - pQueryHandle->cur = save; - return false; + } else { // check data in cache + return hasMoreDataInCacheForSingleModel(pQueryHandle); } + } else { + // next block in the same file + cur->slot += step; + + SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; + cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; + return loadQualifiedDataFromFileBlock(pQueryHandle); } - - // next block in the same file - int32_t fid = cur->fid; -// fileIndex = vnodeGetVnodeHeaderFileIndex(&fid, pQueryHandle->order, &pQueryHandle->vnodeFileInfo); - cur->slot += step; - - SCompBlock *pBlock = &pQueryHandle->pBlock[cur->slot]; - cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; -// return loadQaulifiedData(pQueryHandle); } else { // data in cache return hasMoreDataInCacheForSingleModel(pQueryHandle); } @@ -523,12 +586,12 @@ int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { return midPos; } -static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SArray *sa) { - // only return the qualified data to client in terms of query time window, data rows in the same block but do not - // be included in the query time window will be discarded +// only return the qualified data to client in terms of query time window, data rows in the same block but do not +// be included in the query time window will be discarded +static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCols, SArray *sa) { SQueryFilePos *cur = &pQueryHandle->cur; STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - SDataBlockInfo blockInfo = getTrueBlockInfo(pQueryHandle, pCheckInfo); + SDataBlockInfo blockInfo = getTrueDataBlockInfo(pQueryHandle, pCheckInfo); int32_t endPos = cur->pos; if (QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { @@ -538,7 +601,7 @@ static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SArray *sa) { endPos = 0; pQueryHandle->realNumOfRows = cur->pos + 1; } else { -// endPos = vnodeBinarySearchKey(pQueryHandle->tsBuf->data, blockInfo.size, pQueryHandle->window.ekey, pQueryHandle->order); + endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, pQueryHandle->order); if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { if (endPos < cur->pos) { @@ -560,19 +623,18 @@ static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SArray *sa) { int32_t start = MIN(cur->pos, endPos); // move the data block in the front to data block if needed - if (start != 0) { - int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + + for (int32_t i = 0; i < taosArrayGetSize(sa); ++i) { + int16_t colId = *(int16_t *)taosArrayGet(sa, i); - for (int32_t i = 0; i < taosArrayGetSize(sa); ++i) { - int16_t colId = *(int16_t *)taosArrayGet(sa, i); + for (int32_t j = 0; j < numOfCols; ++j) { + SColumnInfoEx *pCol = taosArrayGet(pQueryHandle->pColumns, j); - for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoEx *pCol = taosArrayGet(pQueryHandle->pColumns, j); - - if (pCol->info.colId == colId) { - memmove(pCol->pData, ((char *)pCol->pData) + pCol->info.bytes * start, pQueryHandle->realNumOfRows * pCol->info.bytes); - break; - } + if (pCol->info.colId == colId) { + SDataCol* pDataCol = &pCols->cols[i]; + memmove(pCol->pData, pDataCol->pData + pCol->info.bytes * start, pQueryHandle->realNumOfRows * pCol->info.bytes); + break; } } } @@ -583,37 +645,134 @@ static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SArray *sa) { cur->pos = endPos; } +static SArray *getColumnIdList(STsdbQueryHandle *pQueryHandle) { + int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + SArray *pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoEx *pCol = taosArrayGet(pQueryHandle->pColumns, i); + taosArrayPush(pIdList, &pCol->info.colId); + } + + return pIdList; +} + +SArray *getDefaultLoadColumns(STsdbQueryHandle *pQueryHandle, bool loadTS) { + SArray *pLocalIdList = getColumnIdList(pQueryHandle); + + // check if the primary time stamp column needs to load + int16_t colId = *(int16_t *)taosArrayGet(pLocalIdList, 0); + + // the primary timestamp column does not be included in the the specified load column list, add it + if (loadTS && colId != 0) { + int16_t columnId = 0; + taosArrayInsert(pLocalIdList, 0, &columnId); + } + + return pLocalIdList; +} + +static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { + int firstPos, lastPos, midPos = -1; + int numOfPoints; + TSKEY *keyList; + + if (num <= 0) return -1; + + keyList = (TSKEY *)pValue; + firstPos = 0; + lastPos = num - 1; + + if (order == 0) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[lastPos]) return lastPos; + if (key == keyList[firstPos]) return firstPos; + if (key < keyList[firstPos]) return firstPos - 1; + + numOfPoints = lastPos - firstPos + 1; + midPos = (numOfPoints >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfPoints = lastPos - firstPos + 1; + midPos = (numOfPoints >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); - SFileGroup* fileGroup = tsdbSearchFGroup(pFileHandle, fid); - if (fileGroup == NULL) { - return false; - } + tsdbInitFileGroupIter(pFileHandle, &pCheckInfo->fileIter, TSDB_FGROUP_ITER_FORWARD); + tsdbSeekFileGroupIter(&pCheckInfo->fileIter, fid); + pCheckInfo->pFileGroup = tsdbGetFileGroupNext(&pCheckInfo->fileIter); SQueryFilePos* cur = &pQueryHandle->cur; - + TSKEY key = pCheckInfo->lastKey; int32_t index = -1; - // todo add iterator for filegroup - while (1) { - if ((fid = getFileCompInfo(pCheckInfo, fileGroup)) < 0) { + int32_t tid = pCheckInfo->tableId.tid; + + while (pCheckInfo->pFileGroup != NULL) { + if ((fid = getFileCompInfo(pCheckInfo, pCheckInfo->pFileGroup)) < 0) { break; } + + SFile* pFile = &pCheckInfo->pFileGroup->files[TSDB_FILE_TYPE_DATA]; + + // no data block in current file, try next + if (pCheckInfo->compIndex[tid].numOfSuperBlocks == 0) { + dTrace("QInfo:%p no data block in file, fid:%d, tid:%d, try next", pQueryHandle->qinfo, + pCheckInfo->pFileGroup->fileId, tid); + + pCheckInfo->pFileGroup = tsdbGetFileGroupNext(&pCheckInfo->fileIter); + + continue; + } - int32_t tid = pCheckInfo->tableId.tid; - index = binarySearchForBlockImpl(pCheckInfo->pBlock, pCheckInfo->compIndex[tid].numOfSuperBlocks, pQueryHandle->order, key); + index = binarySearchForBlockImpl(pCheckInfo->pCompInfo->blocks, pCheckInfo->compIndex[tid].numOfSuperBlocks, pQueryHandle->order, key); if (type == QUERY_RANGE_GREATER_EQUAL) { - if (key <= pCheckInfo->pBlock[index].keyLast) { + if (key <= pCheckInfo->pCompInfo->blocks[index].keyLast) { break; } else { index = -1; } } else { - if (key >= pCheckInfo->pBlock[index].keyFirst) { + if (key >= pCheckInfo->pCompInfo->blocks[index].keyFirst) { break; } else { index = -1; @@ -626,19 +785,33 @@ static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInf return false; } - assert(index >= 0 && index < pQueryHandle->numOfBlocks); + assert(index >= 0 && index < pCheckInfo->compIndex[tid].numOfSuperBlocks); // load first data block into memory failed, caused by disk block error bool blockLoaded = false; - SArray *sa = NULL; + SArray *sa = getDefaultLoadColumns(pQueryHandle, true); // todo no need to loaded at all cur->slot = index; -// sa = getDefaultLoadColumns(pQueryHandle, true); - if (tsdbLoadDataBlock(&fileGroup->files[2], &pCheckInfo->pBlock[cur->slot], 1, fid, sa) == 0) { - blockLoaded = true; - } + SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; + SCompData* data = calloc(1, sizeof(SCompData)+ sizeof(SCompCol)*pBlock->numOfCols); + + data->numOfCols = pBlock->numOfCols; + data->uid = pCheckInfo->pTableObj->tableId.uid; + + pCheckInfo->pDataCols = tdNewDataCols(1000, 2, 4096); + tdInitDataCols(pCheckInfo->pDataCols, pCheckInfo->pTableObj->schema); + + SFile* pFile = &pCheckInfo->pFileGroup->files[TSDB_FILE_TYPE_DATA]; + if (pFile->fd == FD_INITIALIZER) { + pFile->fd = open(pFile->fname, O_RDONLY); + } + + if (tsdbLoadDataBlock(pFile, &pCheckInfo->pCompInfo->blocks[cur->slot], 1, + pCheckInfo->pDataCols, data) == 0) { + blockLoaded = true; + } // dError("QInfo:%p fileId:%d total numOfBlks:%d blockId:%d load into memory failed due to error in disk files", // GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx); @@ -649,10 +822,13 @@ static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInf } // todo search qualified points in blk, according to primary key (timestamp) column -// cur->pos = binarySearchForBlockImpl(ptsBuf->data, pBlocks->numOfPoints, key, pQueryHandle->order); + SDataCols* pDataCols = pCheckInfo->pDataCols; + cur->pos = binarySearchForKey(pDataCols->cols[0].pData, pBlock->numOfPoints, key, pQueryHandle->order); + + cur->fid = pCheckInfo->pFileGroup->fileId; assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); - filterDataInDataBlock(pQueryHandle, sa); + filterDataInDataBlock(pQueryHandle, pCheckInfo->pDataCols, sa); return pQueryHandle->realNumOfRows > 0; } @@ -755,32 +931,46 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - STableCheckInfo* pTableQInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - STable *pTable = pTableQInfo->pTableObj; + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + STable *pTable = pCheckInfo->pTableObj; TSKEY skey = 0, ekey = 0; int32_t rows = 0; - if (pTable->mem != NULL) { - - // create mem table iterator if it is not created yet - if (pTableQInfo->iter == NULL) { - pTableQInfo->iter = tSkipListCreateIter(pTable->mem->pData); + // data in file + if (pHandle->cur.fid > 0) { + SDataBlockInfo binfo = getTrueDataBlockInfo(pHandle, pCheckInfo); + if (binfo.size == pHandle->realNumOfRows) { + return binfo; + } else { + /* not a whole disk block, only the qualified rows, so this block is loaded in to buffer during the + * block next function + */ + SColumnInfoEx* pColInfoEx = taosArrayGet(pHandle->pColumns, 0); + + rows = pHandle->realNumOfRows; + skey = *(TSKEY*) pColInfoEx->pData; + ekey = *(TSKEY*) pColInfoEx->pData + TSDB_KEYSIZE * (rows - 1); + } + } else { + if (pTable->mem != NULL) { + // create mem table iterator if it is not created yet + if (pCheckInfo->iter == NULL) { + pCheckInfo->iter = tSkipListCreateIter(pTable->mem->pData); + } + rows = tsdbReadRowsFromCache(pCheckInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle); } - - rows = tsdbReadRowsFromCache(pTableQInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle); } SDataBlockInfo blockInfo = { - .uid = pTable->tableId.uid, - .sid = pTable->tableId.tid, - .size = rows, - .window = {.skey = skey, .ekey = ekey} + .uid = pTable->tableId.uid, + .sid = pTable->tableId.tid, + .size = rows, + .window = {.skey = skey, .ekey = ekey} }; // update the last key value - pTableQInfo->lastKey = ekey + 1; - + pCheckInfo->lastKey = ekey + 1; return blockInfo; } @@ -791,9 +981,30 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SData } SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList) { - // in case of data in cache, all data has been kept in column info object. + /** + * In the following two cases, the data has been loaded to SColumnInfoEx. + * 1. data is from cache, 2. data block is not completed qualified to query time range + */ STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - return pHandle->pColumns; + + if (pHandle->cur.fid < 0) { + return pHandle->pColumns; + } else { + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + SDataBlockInfo binfo = getTrueDataBlockInfo(pHandle, pCheckInfo); + assert(pHandle->realNumOfRows <= binfo.size); + + if (pHandle->realNumOfRows < binfo.size) { + return pHandle->pColumns; + } else { + SArray *sa = getDefaultLoadColumns(pHandle, true); + + doLoadDataFromFileBlock(pHandle); + filterDataInDataBlock(pHandle, pCheckInfo->pDataCols, sa); + return pHandle->pColumns; + } + } } int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow *window, tsdbpos_t position, int16_t order) {} @@ -1011,19 +1222,20 @@ static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pS } void filterPrepare(void* expr, void* param) { - tSQLBinaryExpr *pExpr = (tSQLBinaryExpr*) expr; - if (pExpr->info != NULL) { + tExprNode *pExpr = (tExprNode*) expr; + if (pExpr->_node.info != NULL) { return; } int32_t i = 0, offset = 0; - pExpr->info = calloc(1, sizeof(tQueryInfo)); + pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); + + tQueryInfo* pInfo = pExpr->_node.info; - tQueryInfo* pInfo = pExpr->info; SSyntaxTreeFilterSupporter* pSupporter = (SSyntaxTreeFilterSupporter*)param; - tVariant* pCond = pExpr->pRight->pVal; - SSchema* pSchema = pExpr->pLeft->pSchema; + tVariant* pCond = pExpr->_node.pRight->pVal; + SSchema* pSchema = pExpr->_node.pLeft->pSchema; getTagColumnInfo(pSupporter, pSchema, &i, &offset); assert((i >= 0 && i < TSDB_MAX_TAGS) || (i == TSDB_TBNAME_COLUMN_INDEX)); @@ -1031,7 +1243,7 @@ void filterPrepare(void* expr, void* param) { pInfo->sch = *pSchema; pInfo->colIdx = i; - pInfo->optr = pExpr->nSQLBinaryOptr; + pInfo->optr = pExpr->_node.optr; pInfo->offset = offset; pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr); @@ -1089,7 +1301,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { static int32_t doQueryTableList(STable* pSTable, SArray* pRes, const char* pCond) { STColumn* stcol = schemaColAt(pSTable->tagSchema, 0); - tSQLBinaryExpr* pExpr = NULL; + tExprNode* pExpr = NULL; tSQLBinaryExprFromString(&pExpr, stcol, schemaNCols(pSTable->tagSchema), pCond, strlen(pCond)); // failed to build expression, no result, return immediately @@ -1110,7 +1322,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, const char* pCond }; tSQLBinaryExprTraverse(pExpr, pSTable->pIndex, pRes, &supp); - tSQLBinaryExprDestroy(&pExpr, tSQLListTraverseDestroyInfo); + tExprTreeDestroy(&pExpr, tSQLListTraverseDestroyInfo); tansformQueryResult(pRes); diff --git a/tests/script/general/show/dnodes.sim b/tests/script/general/show/dnodes.sim new file mode 100644 index 0000000000000000000000000000000000000000..df78194365b091811c9472f5b31995a3c7e10194 --- /dev/null +++ b/tests/script/general/show/dnodes.sim @@ -0,0 +1,16 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== unsupport +sql_error create dnode 192.168.0.2 +sql_error drop dnode 192.168.0.2 + +print =============== show dnodes +sql show dnodes; +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02