diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 366faa15dca19392386683dcb499b8dc5b97334a..985a6741f95ded4e3f430e5ee656434d2f4cef42 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -514,7 +514,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscTansformSQLFunctionForSTableQuery(pQueryInfo); tscIncStreamExecutionCount(pSql->pStream); } else { - tscTrace("%p get tableMeta/metricMeta successfully", pSql); + tscTrace("%p get tableMeta successfully", pSql); } tscDoQuery(pSql); diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index d69f6d295f13bb9515b3c5800f4c4b4bb0764daa..1d4ed51f32a68cb39d511f83c5ea1f47a3aeb4eb 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -605,7 +605,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); -// (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pTableMetaInfo->pMetricMeta->numOfVnodes); + (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * 1); if (*pMemBuffer == NULL) { tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -636,10 +636,11 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); -// for (int32_t i = 0; i < pTableMetaInfo->pMetricMeta->numOfVnodes; ++i) { -// (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); -// (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; -// } + size_t numOfSubs = taosArrayGetSize(pTableMetaInfo->vgroupIdList); + for (int32_t i = 0; i < numOfSubs; ++i) { + (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); + (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; + } if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) { pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 498a33838966bd4de795f0ab9474cd71bc83988c..111ebc4e8c690cef9c52ea214dacde0cf5d58dfd 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -509,31 +509,31 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSubmitMsg *pShellMsg; - char * pMsg, *pStart; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - pStart = pSql->cmd.payload + tsRpcHeadSize; - pMsg = pStart; + char* pMsg = pSql->cmd.payload + tsRpcHeadSize; + + // NOTE: shell message size should not include SMsgDesc + int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc); SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg; - pMsgDesc->numOfVnodes = htonl(1); //set the number of vnodes + + pMsgDesc->numOfVnodes = htonl(1); //todo set the right number of vnodes pMsg += sizeof(SMsgDesc); - pShellMsg = (SSubmitMsg *)pMsg; + SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg; + pShellMsg->header.vgId = htonl(pTableMeta->vgId); - pShellMsg->header.contLen = htonl(pSql->cmd.payloadLen); + pShellMsg->header.contLen = htonl(size); pShellMsg->length = pShellMsg->header.contLen; pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted - // pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here + // pSql->cmd.payloadLen is set during copying data into paylaod pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; + tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htons(pMsgDesc->numOfVnodes)); -// tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip), -// htons(pShellMsg->vnode)); return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 21ce270466f19396f095124eb1818d4b48a866ba..b0c7b68ab4cfc263f6728d4ff8e95c224ec988cf 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1006,13 +1006,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pRes->qhandle = 1; // hack the qhandle check - const uint32_t nBufferSize = (1 << 16); // 64KB + const uint32_t nBufferSize = (1u << 16); // 64KB SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfSubQueries = 0; -// int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes; + int32_t numOfSubQueries = taosArrayGetSize(pTableMetaInfo->vgroupIdList); assert(numOfSubQueries > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); @@ -1118,7 +1117,7 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { tfree(trsupport); } -static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); +static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) { // set no disk space error info @@ -1140,7 +1139,7 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES pthread_mutex_unlock(&trsupport->queryMutex); - tscRetrieveFromVnodeCallBack(trsupport, tres, trsupport->pState->code); + tscRetrieveFromDnodeCallBack(trsupport, tres, trsupport->pState->code); } static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -1235,7 +1234,86 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq } } -static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { +static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { + int32_t idx = trsupport->subqueryIndex; + SSqlObj * pPObj = trsupport->pParentSqlObj; + tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; + + SSubqueryState* pState = trsupport->pState; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + + // data in from current vnode is stored in cache and disk + uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; +// tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, +// pSvd->vnode, numOfRowsFromSubquery, idx); + + tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); + +#ifdef _DEBUG_VIEW + printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems); + SSrcColumnInfo colInfo[256] = {0}; + tscGetSrcColumnInfo(colInfo, pQueryInfo); + tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems, + trsupport->localBuffer->numOfElems, colInfo); +#endif + + if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { + tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, + tsAvailTmpDirGB, tsMinimalTmpDirGB); + tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE); + return; + } + + // each result for a vnode is ordered as an independant list, + // then used as an input of loser tree for disk-based merge routine + int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, + pQueryInfo->groupbyExpr.orderType); + if (ret != 0) { // set no disk space error info, and abort retry + return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE); + } + + // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion + // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. + // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. + int32_t numOfTotal = pState->numOfTotal; + + int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); + if (finished < numOfTotal) { + tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); + return tscFreeSubSqlObj(trsupport, pSql); + } + + // all sub-queries are returned, start to local merge process + pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; + + tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, + pState->numOfTotal, pState->numOfRetrievedRows); + + SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); + tscClearInterpInfo(pPQueryInfo); + + tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, + &pPObj->cmd, &pPObj->res); + tscTrace("%p build loser tree completed", pPObj); + + pPObj->res.precision = pSql->res.precision; + pPObj->res.numOfRows = 0; + pPObj->res.row = 0; + + // only free once + tfree(trsupport->pState); + tscFreeSubSqlObj(trsupport, pSql); + + // set the command flag must be after the semaphore been correctly set. + pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; + if (pPObj->res.code == TSDB_CODE_SUCCESS) { + (*pPObj->fp)(pPObj->param, pPObj, 0); + } else { + tscQueueAsyncRes(pPObj); + } +} + +static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { SRetrieveSupport *trsupport = (SRetrieveSupport *)param; int32_t idx = trsupport->subqueryIndex; SSqlObj * pPObj = trsupport->pParentSqlObj; @@ -1264,15 +1342,15 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); - SVnodeSidList *vnodeInfo = 0; - SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; +// SVnodeSidList *vnodeInfo = 0; +// SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; if (numOfRows > 0) { assert(pRes->numOfRows == numOfRows); int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, - pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); +// tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, +// pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64, @@ -1298,85 +1376,16 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data, pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); - if (ret < 0) { - // set no disk space error info, and abort retry + if (ret < 0) { // set no disk space error info, and abort retry tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - } else { + } else if (pRes->completed) { + tscAllDataRetrievedFromDnode(trsupport, pSql); + } else { // continue fetch data from dnode pthread_mutex_unlock(&trsupport->queryMutex); - taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); - } - - } else { // all data has been retrieved to client - /* data in from current vnode is stored in cache and disk */ - uint32_t numOfRowsFromVnode = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; - tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, - pSvd->vnode, numOfRowsFromVnode, idx); - - tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); - -#ifdef _DEBUG_VIEW - printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems); - SSrcColumnInfo colInfo[256] = {0}; - tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems, - trsupport->localBuffer->numOfElems, colInfo); -#endif - - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, - tsAvailTmpDirGB, tsMinimalTmpDirGB); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - return; - } - - // each result for a vnode is ordered as an independant list, - // then used as an input of loser tree for disk-based merge routine - int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, - pQueryInfo->groupbyExpr.orderType); - if (ret != 0) { - /* set no disk space error info, and abort retry */ - return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - } - - // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion - // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. - // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. - int32_t numOfTotal = pState->numOfTotal; - - int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); - if (finished < numOfTotal) { - tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); - return tscFreeSubSqlObj(trsupport, pSql); - } - - // all sub-queries are returned, start to local merge process - pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; - - tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, - pState->numOfTotal, pState->numOfRetrievedRows); - - SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); - tscClearInterpInfo(pPQueryInfo); - - tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, - &pPObj->cmd, &pPObj->res); - tscTrace("%p build loser tree completed", pPObj); - - pPObj->res.precision = pSql->res.precision; - pPObj->res.numOfRows = 0; - pPObj->res.row = 0; - - // only free once - tfree(trsupport->pState); - tscFreeSubSqlObj(trsupport, pSql); - - // set the command flag must be after the semaphore been correctly set. - pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - if (pPObj->res.code == TSDB_CODE_SUCCESS) { - (*pPObj->fp)(pPObj->param, pPObj, 0); - } else { - tscQueueAsyncRes(pPObj); + taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } + } else { // all data has been retrieved to client + tscAllDataRetrievedFromDnode(trsupport, pSql); } } @@ -1436,7 +1445,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { /* * if a query on a vnode is failed, all retrieve operations from vnode that occurs later - * than this one are actually not necessary, we simply call the tscRetrieveFromVnodeCallBack + * than this one are actually not necessary, we simply call the tscRetrieveFromDnodeCallBack * function to abort current and remain retrieve process. * * NOTE: threadsafe is required. @@ -1474,7 +1483,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->subqueryIndex, pState->code); } - tscRetrieveFromVnodeCallBack(param, tres, pState->code); + tscRetrieveFromDnodeCallBack(param, tres, pState->code); } else { // success, proceed to retrieve data from dnode if (vnodeInfo != NULL) { tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, @@ -1485,7 +1494,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->subqueryIndex); } - taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); + taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 18f491aa9edaaa60efbc6cbca6bfc54e532f8b9c..ec6881db3f5ae4f1e32dbf790bfb662ae3df84c5 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -608,7 +608,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { * the payloadLen should be actual message body size * the old value of payloadLen is the allocated payload size */ - pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize - sizeof(SMsgDesc); + pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize; assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100 && pCmd->payloadLen > 0); return TSDB_CODE_SUCCESS; diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1f53d50c1f3b4f57571508060db4f19616eb49de --- /dev/null +++ b/src/client/tests/timeParseTest.cpp @@ -0,0 +1,167 @@ +#include +#include +#include + +#include "taos.h" +#include "tstoken.h" +#include "ttime.h" +#include "tutil.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +extern void deltaToUtcInitOnce(); +/* test parse time function */ +TEST(testCase, parse_time) { + + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + deltaToUtcInitOnce(); + + char t1[] = "2018-1-1 1:1:1.952798"; + char t13[] = "1970-1-1 0:0:0"; + + int64_t time = 0, time1 = 0; + + taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); + + char t2[] = "2018-1-1T1:1:1.952Z"; + taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI); + + EXPECT_EQ(time, 1514739661952 + 28800000); + + char t3[] = "2018-1-1 1:01:01.952"; + taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + char t4[] = "2018-1-1 1:01:01.9"; + char t5[] = "2018-1-1 1:01:1.900"; + char t6[] = "2018-01-01 1:1:1.90"; + char t7[] = "2018-01-01 01:01:01.9"; + char t8[] = "2018-01-01 01:01:01.9007865"; + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t9[] = "2017-4-3 1:1:2.980"; + char t10[] = "2017-4-3T2:1:2.98+9:00"; + taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t11[] = "2017-4-3T2:1:2.98+09:00"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t12[] = "2017-4-3T2:1:2.98+0900"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taos_options(TSDB_OPTION_TIMEZONE, "UTC"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + deltaToUtcInitOnce(); + + char t14[] = "1970-1-1T0:0:0Z"; + taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + char t40[] = "1970-1-1 0:0:0.999999999"; + taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); + + char t41[] = "1997-1-1 0:0:0.999999999"; + taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999); + + int64_t k = timezone; + char t42[] = "1997-1-1T0:0:0.999999999Z"; + taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); + + //////////////////////////////////////////////////////////////////// + // illegal timestamp format + char t15[] = "2017-12-33 0:0:0"; + EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1); + + char t16[] = "2017-12-31 99:0:0"; + EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1); + + char t17[] = "2017-12-31T9:0:0"; + EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1); + + char t18[] = "2017-12-31T9:0:0.Z"; + EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1); + + char t19[] = "2017-12-31 9:0:0.-1"; + EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1); + + char t20[] = "2017-12-31 9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0); + EXPECT_EQ(time, 1514682000100); + + char t21[] = "2017-12-31T9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1); + + char t22[] = "2017-12-31 9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0); + + char t23[] = "2017-12-31T9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0); + + + //======================== add some case ============================// + + char b1[] = "9999-12-31 23:59:59.999"; + taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 253402271999999); + + + char b2[] = "2020-01-01 01:01:01.321"; + taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1577811661321); + + taos_options(TSDB_OPTION_TIMEZONE, "America/New_York"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND); +} + + diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 9ac06e4199becba6759f6279d8622ba88475b9b8..4632c67c3cd5dc65267a151f63dcad40cea24b1f 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -188,15 +188,6 @@ extern char *taosMsg[]; #pragma pack(push, 1) -// typedef struct { -// int32_t vnode; -// int32_t sid; -// int32_t sversion; -// uint64_t uid; -// int16_t numOfRows; -// char payLoad[]; -//} SShellSubmitBlock; - typedef struct { int32_t numOfVnodes; } SMsgDesc; diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index 0e51962f49999ae55c1c18aa5acd9d9baba64be7..a6462f98558d163b05d3b8b1cbc2cad23ae7cae4 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -12,4 +12,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(query ${SRC}) TARGET_LINK_LIBRARIES(query tsdb tutil m rt) -ENDIF () \ No newline at end of file +ENDIF () + +ADD_SUBDIRECTORY(tests) \ No newline at end of file diff --git a/src/query/inc/qtsbuf.h b/src/query/inc/qtsbuf.h index 1afdb0cd6cb9175b14675d06446afbe3a891df27..8e014e5feb781b2c87209118d1c4c8a7c2d13813 100644 --- a/src/query/inc/qtsbuf.h +++ b/src/query/inc/qtsbuf.h @@ -48,10 +48,10 @@ typedef struct STSElem { } STSElem; typedef struct STSCursor { - int32_t vnodeIndex; - int32_t blockIndex; - int32_t tsIndex; - int32_t order; + int32_t vnodeIndex; + int32_t blockIndex; + int32_t tsIndex; + uint32_t order; } STSCursor; typedef struct STSBlock { diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 2e1d86a94261cfe692411fcd5ce0b566423aa991..5adce04efa44d7e85bc9ccf482dd21e0deff20b5 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -32,12 +32,6 @@ typedef struct SData { char data[]; } SData; -enum { -// ST_QUERY_KILLED = 0, // query killed - ST_QUERY_PAUSED = 1, // query paused, due to full of the response buffer - ST_QUERY_COMPLETED = 2, // query completed -}; - struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); @@ -60,18 +54,20 @@ typedef struct SWindowStatus { } SWindowStatus; typedef struct SWindowResult { - uint16_t numOfRows; + uint16_t numOfRows; // number of rows of current time window SPosInfo pos; // Position of current result in disk-based output buffer SResultInfo* resultInfo; // For each result column, there is a resultInfo STimeWindow window; // The time window that current result covers. - SWindowStatus status; + SWindowStatus status; // this result status: closed or opened } SWindowResult; typedef struct SResultRec { - int64_t total; - int64_t size; - int64_t capacity; - int32_t threshold; // the threshold size, when the number of rows in result buffer, return to client + int64_t total; // total generated result size in rows + int64_t size; // current result set size in rows + int64_t capacity; // capacity of current result output buffer + + // result size threshold in rows. If the result buffer is larger than this, pause query and return to client + int32_t threshold; } SResultRec; typedef struct SWindowResInfo { @@ -99,7 +95,6 @@ typedef struct SSingleColumnFilterInfo { void* pData; } SSingleColumnFilterInfo; -/* intermediate pos during multimeter query involves interval */ typedef struct STableQueryInfo { int64_t lastKey; STimeWindow win; @@ -107,7 +102,7 @@ typedef struct STableQueryInfo { int16_t queryRangeSet; // denote if the query range is set, only available for interval query int64_t tag; STSCursor cur; - int32_t sid; // for retrieve the page id list + int32_t tid; // for retrieve the page id list SWindowResInfo windowResInfo; } STableQueryInfo; @@ -116,7 +111,6 @@ typedef struct STableDataInfo { int32_t numOfBlocks; int32_t start; // start block index int32_t tableIndex; - void* pMeterObj; int32_t groupIdx; // group id in table list STableQueryInfo* pTableQInfo; } STableDataInfo; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 2f219368080b135cb380293f565dfb332320b538..83cbae42664407d1245adfc9740fa31be5af1de0 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -375,29 +375,6 @@ bool doRevisedResultsByLimit(SQInfo *pQInfo) { return false; } -/** - * - * @param pQuery - * @param pDataBlockInfo - * @param forwardStep - * @return TRUE means query not completed, FALSE means query is completed - */ -static bool queryPaused(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, int32_t forwardStep) { - // output buffer is full, pause current query - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - // assert((QUERY_IS_ASC_QUERY(pQuery) && forwardStep + pQuery->pos <= pDataBlockInfo->size) || - // (!QUERY_IS_ASC_QUERY(pQuery) && pQuery->pos - forwardStep + 1 >= 0)); - // - return true; - } - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - return true; - } - - return false; -} - static bool isTopBottomQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; @@ -1690,7 +1667,7 @@ bool notHasQueryTimeRange(SQuery *pQuery) { (pQuery->window.skey == INT64_MAX && pQuery->window.ekey == 0 && (!QUERY_IS_ASC_QUERY(pQuery))); } -bool needSupplementaryScan(SQuery *pQuery) { +static bool needReverseScan(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { @@ -2227,7 +2204,7 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { size_t s = taosArrayGetSize(pQInfo->pTableIdList); num = MAX(s, INITIAL_RESULT_ROWS_VALUE); } else { // for super table query, one page for each subset - // num = pQInfo->pSidSet->numOfSubSet; + num = 1;//pQInfo->pSidSet->numOfSubSet; } assert(num > 0); @@ -2664,7 +2641,6 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfRes = 0; SDataStatis *pStatis = NULL; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &numOfRes, &pRuntimeEnv->windowResInfo, pDataBlock); @@ -2950,7 +2926,7 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) return leftTimestamp > rightTimestamp ? 1 : -1; } -int32_t mergeMetersResultToOneGroups(SQInfo *pQInfo) { +int32_t mergeResultsToGroup(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -2990,7 +2966,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { pQInfo->numOfGroupResultPages = 0; // current results of group has been sent to client, try next group - if (mergeMetersResultToOneGroups(pQInfo) != TSDB_CODE_SUCCESS) { + if (mergeResultsToGroup(pQInfo) != TSDB_CODE_SUCCESS) { return; // failed to save data in the disk } @@ -3071,9 +3047,9 @@ int32_t doMergeMetersResultsToGroupRes(SQInfo *pQInfo, STableDataInfo *pTableDat // todo opt for the case of one table per group int32_t numOfMeters = 0; for (int32_t i = start; i < end; ++i) { - int32_t sid = pTableDataInfo[i].pTableQInfo->sid; + int32_t tid = pTableDataInfo[i].pTableQInfo->tid; - SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, sid); + SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, tid); if (list.size > 0 && pTableDataInfo[i].pTableQInfo->windowResInfo.size > 0) { pTableList[numOfMeters] = &pTableDataInfo[i]; numOfMeters += 1; @@ -3240,10 +3216,9 @@ void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pRes } } -void setMeterDataInfo(STableDataInfo *pTableDataInfo, void *pMeterObj, int32_t meterIdx, int32_t groupId) { - pTableDataInfo->pMeterObj = pMeterObj; +void setTableDataInfo(STableDataInfo *pTableDataInfo, int32_t tableIndex, int32_t groupId) { pTableDataInfo->groupIdx = groupId; - pTableDataInfo->tableIndex = meterIdx; + pTableDataInfo->tableIndex = tableIndex; } static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo *pWindowResInfo, int32_t order) { @@ -3297,7 +3272,7 @@ void disableFunctForTableSuppleScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order pQuery->order.order = pQuery->order.order ^ 1u; } -void disableFunctForSuppleScan(SQInfo *pQInfo, int32_t order) { +void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -3322,7 +3297,7 @@ void disableFunctForSuppleScan(SQInfo *pQInfo, int32_t order) { pQuery->order.order = (pQuery->order.order) ^ 1u; } -void enableFunctForMasterScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { +void enableFuncForForwardScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { @@ -3483,7 +3458,7 @@ static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { SQuery * pQuery = pRuntimeEnv->pQuery; SQueryStatus qStatus = {0}; - if (!needSupplementaryScan(pQuery)) { + if (!needReverseScan(pQuery)) { return; } @@ -3503,7 +3478,7 @@ static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { doScanAllDataBlocks(pRuntimeEnv); queryStatusRestore(pRuntimeEnv, &qStatus); - enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + enableFuncForForwardScan(pRuntimeEnv, pQuery->order.order); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } @@ -3562,7 +3537,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { return toContinue; } -void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { +void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); @@ -3588,10 +3563,7 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { break; } - /* - * set the correct start position, and load the corresponding block in buffer for next - * round scan all data blocks. - */ + // set the correct start position, and load the corresponding block in buffer for next round scan all data blocks. int32_t ret = tsdbDataBlockSeek(pRuntimeEnv->pQueryHandle, pos); status = pQuery->status; @@ -3675,18 +3647,13 @@ static bool hasMainOutput(SQuery *pQuery) { return false; } -STableQueryInfo *createMeterQueryInfo(SQInfo *pQInfo, int32_t sid, TSKEY skey, TSKEY ekey) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - +STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, int32_t tid, STimeWindow win) { STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); - pTableQueryInfo->win = (STimeWindow){ - .skey = skey, - .ekey = ekey, - }; - pTableQueryInfo->lastKey = skey; + pTableQueryInfo->win = win; + pTableQueryInfo->lastKey = win.skey; - pTableQueryInfo->sid = sid; + pTableQueryInfo->tid = tid; pTableQueryInfo->cur.vnodeIndex = -1; initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, 100, 100, TSDB_DATA_TYPE_INT); @@ -3702,7 +3669,7 @@ void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) free(pTableQueryInfo); } -void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo, TSKEY skey, TSKEY ekey) { +void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { if (pTableQueryInfo == NULL) { return; } @@ -3906,7 +3873,7 @@ static int32_t getNumOfSubset(SQInfo *pQInfo) { if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (isIntervalQuery(pQuery))) { totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo); } else { - // totalSubset = pQInfo->pSidSet->numOfSubSet; + totalSubset = 1;//pQInfo->pSidSet->numOfSubSet; } return totalSubset; @@ -3942,19 +3909,18 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; int32_t oldOffset = pQInfo->offset; - assert(0); /* * current output space is not enough to keep all the result data of this group, only copy partial results * to SQuery object's result buffer */ - // if (numOfRowsToCopy > pQuery->pointsToRead - numOfResult) { - // numOfRowsToCopy = pQuery->pointsToRead - numOfResult; - // pQInfo->offset += numOfRowsToCopy; - // } else { - // pQInfo->offset = 0; - // pQInfo->subgroupIdx += 1; - // } + if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { + numOfRowsToCopy = pQuery->rec.capacity - numOfResult; + pQInfo->offset += numOfRowsToCopy; + } else { + pQInfo->offset = 0; + pQInfo->subgroupIdx += 1; + } for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; @@ -3965,13 +3931,12 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde } numOfResult += numOfRowsToCopy; - assert(0); - // if (numOfResult == pQuery->rec.pointsToRead) { - // break; - // } + if (numOfResult == pQuery->rec.capacity) { + break; + } } - dTrace("QInfo:%p copy data to SQuery buf completed", GET_QINFO_ADDR(pQuery)); + dTrace("QInfo:%p copy data to SQuery buf completed", pQInfo); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pQuery, numOfResult); @@ -4014,9 +3979,8 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInf } } -void stableApplyFunctionsOnBlock_(SQInfo *pQInfo, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, +void stableApplyFunctionsOnBlock(SQueryRuntimeEnv* pRuntimeEnv, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray *pDataBlock, __block_search_fn_t searchFn) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; @@ -4169,10 +4133,11 @@ int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage } void vnodePrintQueryStatistics(SQInfo *pQInfo) { +#if 0 SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; -#if 0 + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; if (pRuntimeEnv->pResultBuf == NULL) { pSummary->tmpBufferInDisk = 0; @@ -4213,41 +4178,30 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t initQInfo(SQInfo *pQInfo, void *param, void* tsdb) { +int32_t doInitializeQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t code = TSDB_CODE_SUCCESS; - // only the successful complete requries the sem_post/over = 1 operations. - if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || - (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { - dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, - pQuery->window.ekey, pQuery->order.order); - - sem_post(&pQInfo->dataReady); - setQueryStatus(pQuery, QUERY_COMPLETED); - return TSDB_CODE_SUCCESS; - } - setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQuery, false); // dataInCache requires lastKey value pQuery->lastKey = pQuery->window.skey; - STsdbQueryCond cond = {0}; - - cond.twindow = (STimeWindow){.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; - cond.order = pQuery->order.order; - cond.colList = *pQuery->colList; + STsdbQueryCond cond = { + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = *pQuery->colList, + }; SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); for (int32_t i = 0; i < pQuery->numOfCols; ++i) { taosArrayPush(cols, &pQuery->colList[i]); } - - pQInfo->runtimeEnv.pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); - - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; @@ -4258,15 +4212,34 @@ int32_t initQInfo(SQInfo *pQInfo, void *param, void* tsdb) { } // create runtime environment - code = setupQueryRuntimeEnv(&pQInfo->runtimeEnv, NULL, pQuery->order.order, false); + code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { return code; } - pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, false); - if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, isSTableQuery); + + if (isSTableQuery) { + int32_t rows = getInitialPageNum(pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pQuery->intervalTime == 0) { + int16_t type = TSDB_DATA_TYPE_NULL; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group by columns not tags; + type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + } else { + type = TSDB_DATA_TYPE_INT; // group id + } + + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); + } + + } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); if (code != TSDB_CODE_SUCCESS) { return code; @@ -4363,59 +4336,56 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { } } -static void queryOnDataBlocks(SQInfo *pQInfo, STableDataInfo *pMeterDataInfo) { +static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - - // dTrace("QInfo:%p start to check data blocks in %d files", pQInfo, pVnodeFileInfo->numOfFiles); + + int64_t st = taosGetTimestampMs(); + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + tsdb_query_handle_t *pQueryHandle = pRuntimeEnv->pQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(pQInfo)) { break; } - // prepare the STableDataInfo struct for each table - SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - // SMeterObj * pMeterObj = getMeterObj(pSupporter->pMetersHashTable, blockInfo.sid); - - // pQInfo->pObj = pMeterObj; - // pRuntimeEnv->pMeterObj = pMeterObj; - - STableDataInfo *pTableDataInfo = NULL; - // for (int32_t i = 0; i < pSupporter->pSidSet->numOfTables; ++i) { - // if (pMeterDataInfo[i].pMeterObj == pMeterObj) { - // pTableDataInfo = &pMeterDataInfo[i]; - // break; - // } - // } + STableDataInfo* pTableDataInfo = NULL; + + // todo opt performance + for(int32_t i = 0; i < numOfTables; ++i) { + if (pQInfo->pTableDataInfo[i].pTableQInfo->tid == blockInfo.sid) { + pTableDataInfo = &pQInfo->pTableDataInfo[i]; + break; + } + } - assert(pTableDataInfo != NULL); + assert(pTableDataInfo != NULL && pTableDataInfo->pTableQInfo != NULL); STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; - if (pTableDataInfo->pTableQInfo == NULL) { - // pTableDataInfo->pTableQInfo = createMeterQueryInfo(pQInfo, pMeterObj->sid, pQuery->skey, pQuery->ekey); - } - restoreIntervalQueryRange(pRuntimeEnv, pTableQueryInfo); SDataStatis *pStatis = NULL; SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); TSKEY nextKey = blockInfo.window.ekey; - if (pQuery->intervalTime == 0) { + if (!isIntervalQuery(pQuery)) { setExecutionContext(pQInfo, pTableQueryInfo, pTableDataInfo->tableIndex, pTableDataInfo->groupIdx, nextKey); } else { // interval query setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); int32_t ret = setAdditionalInfo(pQInfo, pTableDataInfo->tableIndex, pTableQueryInfo); + if (ret != TSDB_CODE_SUCCESS) { - // pQInfo->killed = 1; - return; + pQInfo->code = ret; + return taosGetTimestampMs() - st; } } - // stableApplyFunctionsOnBlock_(pSupporter, pTableDataInfo, &blockInfo, pStatis, pDataBlock, searchFn); + stableApplyFunctionsOnBlock(pRuntimeEnv, pTableDataInfo, &blockInfo, pStatis, pDataBlock, binarySearchForKey); } + + int64_t et = taosGetTimestampMs(); + return et - st; } static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *dataInCache, int32_t index, @@ -4499,7 +4469,7 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start pointInterpSupporterSetData(pQInfo, &pointInterpSupporter); pointInterpSupporterDestroy(&pointInterpSupporter); - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); // first/last_row query, do not invoke the finalize for super table query doFinalizeResult(pRuntimeEnv); @@ -4525,9 +4495,10 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start */ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - -#if 0 SQuery* pQuery = pRuntimeEnv->pQuery; + setQueryStatus(pQuery, QUERY_COMPLETED); + +#if 0 // tSidSet *pTableIdList = pSupporter->pSidSet; int32_t vid = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[0]->sid)->vnode; @@ -4679,7 +4650,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { } } - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); pQuery->size = getNumOfResult(pRuntimeEnv); doSkipResults(pRuntimeEnv); @@ -4778,95 +4749,85 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { #endif } -static void doOrderedScan(SQInfo *pQInfo) { - SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; -#if 0 -// if (pQInfo->runtimeEnv. == NULL) { -// pSupporter->pMeterDataInfo = calloc(pSupporter->pSidSet->numOfTables, sizeof(STableDataInfo)); -// } - - STableIdInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo; +static void createTableDataInfo(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - tSidSet* pSidset = pSupporter->pSidSet; - int32_t groupId = 0; + // todo make sure the table are added the reference count to gauranteed that all involved tables are valid + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); - for (int32_t i = 0; i < pSidset->numOfTables; ++i) { // load all meter meta info - SMeterObj *pMeterObj = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[i]->sid); - if (pMeterObj == NULL) { - dError("QInfo:%p failed to find required sid:%d", pQInfo, pMeterSidExtInfo[i]->sid); - continue; + if (pQInfo->pTableDataInfo == NULL) { + pQInfo->pTableDataInfo = (STableDataInfo *)calloc(1, sizeof(STableDataInfo) * numOfTables); + if (pQInfo->pTableDataInfo == NULL) { + dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); + pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; + return; } - if (i >= pSidset->starterPos[groupId + 1]) { - groupId += 1; + int32_t groupId = 0; + for (int32_t i = 0; i < numOfTables; ++i) { // load all meter meta info + STableId *id = taosArrayGet(pQInfo->pTableIdList, i); + STableDataInfo *pInfo = &pQInfo->pTableDataInfo[i]; + + setTableDataInfo(pInfo, i, groupId); + pInfo->pTableQInfo = createTableQueryInfo(&pQInfo->runtimeEnv, id->tid, pQuery->window); } - - STableDataInfo *pOneMeterDataInfo = &pSupporter->pMeterDataInfo[i]; - assert(pOneMeterDataInfo->pMeterObj == NULL); - - setMeterDataInfo(pOneMeterDataInfo, pMeterObj, i, groupId); - pOneMeterDataInfo->pTableQInfo = createMeterQueryInfo(pSupporter, pMeterObj->sid, pQuery->skey, pQuery->ekey); } - - queryOnDataBlocks(pQInfo, pSupporter->pMeterDataInfo); - if (pQInfo->code != TSDB_CODE_SUCCESS) { - return; - } -#endif } -static void setupMeterQueryInfoForSupplementQuery(SQInfo *pQInfo) { +static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - int32_t num = taosHashGetSize(pQInfo->pTableIdList); - for (int32_t i = 0; i < num; ++i) { - // STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; - // changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo, pSupporter->rawSKey, pSupporter->rawEKey); + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + + for (int32_t i = 0; i < numOfTables; ++i) { + STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); } } -static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - - if (!needSupplementaryScan(pQuery)) { - dTrace("QInfo:%p no need to do supplementary scan, query completed", pQInfo); - return; - } - +static void doSaveContext(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); - // disableFunctForSuppleScan(pSupporter, pQuery->order.order); - + disableFuncForReverseScan(pQInfo, pQuery->order.order); + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1u; } - -#if 0 - SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); - setupMeterQueryInfoForSupplementQuery(pSupporter); - - int64_t st = taosGetTimestampMs(); - doOrderedScan(pQInfo); - - int64_t et = taosGetTimestampMs(); - dTrace("QInfo:%p supplementary scan completed, elapsed time: %lldms", pQInfo, et - st); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + prepareQueryInfoForReverseScan(pQInfo); +} + +static void doRestoreContext(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; - /* - * restore the env - * the meter query info is not reset to the original state - */ - SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); - enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; } -#endif + + enableFuncForForwardScan(pRuntimeEnv, pQuery->order.order); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } -static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { +static void doCloseAllTimeWindowAfterScan(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + + if (isIntervalQuery(pQuery)) { + for (int32_t i = 0; i < numOfTables; ++i) { + STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + closeAllTimeWindow(&pTableQueryInfo->windowResInfo); + } + } else { // close results for group result + closeAllTimeWindow(&pQInfo->runtimeEnv.windowResInfo); + } +} + +static void multiTableQueryProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -4875,7 +4836,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { * if the subgroupIdx > 0, the query process must be completed yet, we only need to * copy the data into output buffer */ - if (pQuery->intervalTime > 0) { + if (isIntervalQuery(pQuery)) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW @@ -4894,46 +4855,50 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.size, pQuery->rec.total); return; } -#if 0 - pSupporter->pMeterDataInfo = (STableDataInfo *)calloc(1, sizeof(STableDataInfo) * pSupporter->numOfMeters); - if (pSupporter->pMeterDataInfo == NULL) { - dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); - pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; - return; - } - dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, group:%d", pQInfo, pSupporter->rawSKey, - pSupporter->rawEKey, pQuery->order.order, pSupporter->pSidSet->numOfSubSet); + dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, pQuery->window.skey, + pQuery->window.ekey, pQuery->order.order); - dTrace("QInfo:%p main query scan start", pQInfo); - int64_t st = taosGetTimestampMs(); - doOrderedScan(pQInfo); + // create the query support structures + createTableDataInfo(pQInfo); - int64_t et = taosGetTimestampMs(); - dTrace("QInfo:%p main scan completed, elapsed time: %lldms, supplementary scan start, order:%d", pQInfo, et - st, + // do check all qualified data blocks + int64_t el = queryOnDataBlocks(pQInfo); + dTrace("QInfo:%p forward scan completed, elapsed time: %lldms, reversed scan start, order:%d", pQInfo, el, pQuery->order.order ^ 1u); - if (pQuery->intervalTime > 0) { - for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) { - STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; - closeAllTimeWindow(&pTableQueryInfo->windowResInfo); - } - } else { // close results for group result - closeAllTimeWindow(&pRuntimeEnv->windowResInfo); + // query error occurred or query is killed, abort current execution + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { + dTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); + return; } - doMultiMeterSupplementaryScan(pQInfo); + // close all time window results + doCloseAllTimeWindowAfterScan(pQInfo); - if (isQueryKilled(pQInfo)) { - dTrace("QInfo:%p query killed, abort", pQInfo); + if (needReverseScan(pQuery)) { + doSaveContext(pQInfo); + + el = queryOnDataBlocks(pQInfo); + dTrace("QInfo:%p reversed scan completed, elapsed time: %lldms", pQInfo, el); + + doRestoreContext(pQInfo); + } else { + dTrace("QInfo:%p no need to do reversed scan, query completed", pQInfo); + } + + setQueryStatus(pQuery, QUERY_COMPLETED); + + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { + dTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; } - if (pQuery->intervalTime > 0 || isSumAvgRateQuery(pQuery)) { - assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); + if (isIntervalQuery(pQuery) || isSumAvgRateQuery(pQuery)) { +// assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); - if (mergeMetersResultToOneGroups(pSupporter) == TSDB_CODE_SUCCESS) { - copyResToQueryResultBuf(pSupporter, pQuery); + if (mergeResultsToGroup(pQInfo) == TSDB_CODE_SUCCESS) { + copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); @@ -4944,10 +4909,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { } // handle the limitation of output buffer - pQInfo->size += pQuery->size; - dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->size, pQInfo->size, - pQInfo->pointsReturned); -#endif + dTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); } /* @@ -4960,7 +4922,7 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); if (isQueryKilled(pQInfo)) { @@ -4993,7 +4955,7 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { } while (1) { - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); if (isQueryKilled(pQInfo)) { @@ -5039,7 +5001,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { while (1) { initCtxOutputBuf(pRuntimeEnv); - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return; @@ -5076,7 +5038,7 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { while (1) { tableIntervalProcessImpl(pRuntimeEnv); - if (pQuery->intervalTime > 0) { + if (isIntervalQuery(pQuery)) { pQInfo->subgroupIdx = 0; // always start from 0 pQuery->rec.size = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); @@ -5213,15 +5175,15 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { sem_post(&pQInfo->dataReady); } -void multiTableQueryImpl(SQInfo* pQInfo) { +static void multiTableQueryImpl(SQInfo* pQInfo) { SQuery* pQuery = pQInfo->runtimeEnv.pQuery; pQuery->rec.size = 0; int64_t st = taosGetTimestampUs(); - if (pQuery->intervalTime > 0 || + if (isIntervalQuery(pQuery) || (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { - vnodeMultiMeterQueryProcessor(pQInfo); + multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBufferInLoop == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)); @@ -5242,29 +5204,6 @@ void multiTableQueryImpl(SQInfo* pQInfo) { sem_post(&pQInfo->dataReady); } -void qTableQuery(SQInfo *pQInfo) { - if (pQInfo == NULL || pQInfo->signature != pQInfo) { - dTrace("%p freed abort query", pQInfo); - return; - } - - if (isQueryKilled(pQInfo)) { - dTrace("QInfo:%p it is already killed, abort", pQInfo); - return; - } - - dTrace("QInfo:%p query task is launched", pQInfo); - - int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); - if (numOfTables == 1) { - singleTableQueryImpl(pQInfo); - } else { - multiTableQueryImpl(pQInfo); - } - - // vnodeDecRefCount(pQInfo); -} - static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { int32_t j = 0; @@ -5284,7 +5223,7 @@ bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pEx return j < pQueryMsg->numOfCols; } -static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryMsg) { +static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { if (pQueryMsg->intervalTime < 0) { dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime); return -1; @@ -5316,14 +5255,16 @@ static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryMsg) { static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** pTableIdList) { assert(pQueryMsg->numOfTables > 0); - *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo)); + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableId)); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - taosArrayPush(*pTableIdList, pTableIdInfo); + STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->sid}; + taosArrayPush(*pTableIdList, &id); + pMsg += sizeof(STableIdInfo); for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) { @@ -5349,7 +5290,7 @@ static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, - wchar_t** tagCond, char** nameCond) { + wchar_t** tagCond) { pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); @@ -5374,7 +5315,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); // query msg safety check - if (validateQueryMeterMsg(pQueryMsg) != 0) { + if (validateQueryMsg(pQueryMsg) != 0) { return TSDB_CODE_INVALID_QUERY_MSG; } @@ -5503,10 +5444,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, *tagCond = calloc(1, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); } - - if (pQueryMsg->nameCondLen > 0) { - *nameCond = strndup(pMsg, pQueryMsg->nameCondLen); - } dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, " "timestamp order:%d, tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 @@ -5895,7 +5832,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->pos = -1; - dTrace("QInfo %p is allocated", pQInfo); + pQuery->window.skey = pQueryMsg->window.skey; + pQuery->window.ekey = pQueryMsg->window.ekey; + pQuery->lastKey = pQuery->window.skey; + + if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { + dError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); + goto _clean_memory; + } + + vnodeParametersSafetyCheck(pQuery); + + dTrace("qmsg:%p create QInfo:%p, QInfo created", pQueryMsg, pQInfo); return pQInfo; _clean_memory: @@ -5919,7 +5867,7 @@ _clean_memory: return NULL; } -bool isQInfoValid(void *param) { +static bool isValidQInfo(void *param) { SQInfo *pQInfo = (SQInfo *)param; if (pQInfo == NULL) { return false; @@ -5933,11 +5881,51 @@ bool isQInfoValid(void *param) { return (sig == (uint64_t)pQInfo); } -void vnodeFreeQInfo(SQInfo *pQInfo) { - if (!isQInfoValid(pQInfo)) { - return; +static void freeQInfo(SQInfo *pQInfo); +static int32_t initializeQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { + int32_t code = TSDB_CODE_SUCCESS; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + STSBuf *pTSBuf = NULL; + if (pQueryMsg->tsLen > 0) { // open new file to save the result + char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; + pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); + + tsBufResetPos(pTSBuf); + tsBufNextPos(pTSBuf); + } + + // only the successful complete requries the sem_post/over = 1 operations. + if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || + (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { + dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, + pQuery->window.ekey, pQuery->order.order); + + sem_post(&pQInfo->dataReady); + setQueryStatus(pQuery, QUERY_COMPLETED); + return TSDB_CODE_SUCCESS; + } + + // filter the qualified + if ((code = doInitializeQInfo(pQInfo, pTSBuf, tsdb, isSTable)) != TSDB_CODE_SUCCESS) { + goto _error; } + // dTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, + // pQInfo->refCount); + return code; + +_error: + // table query ref will be decrease during error handling + freeQInfo(pQInfo); + return code; +} + +static void freeQInfo(SQInfo *pQInfo) { + if (!isValidQInfo(pQInfo)) { + return; + } + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; setQueryKilled(pQInfo); @@ -5945,7 +5933,7 @@ void vnodeFreeQInfo(SQInfo *pQInfo) { for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { tfree(pQuery->sdata[col]); } - + // for (int col = 0; col < pQuery->numOfCols; ++col) { // vnodeFreeColumnInfo(&pQuery->colList[col].data); // } @@ -5953,100 +5941,102 @@ void vnodeFreeQInfo(SQInfo *pQInfo) { // if (pQuery->colList[0].colIdx != PRIMARYKEY_TIMESTAMP_COL_INDEX) { // tfree(pQuery->tsData); // } - + sem_destroy(&(pQInfo->dataReady)); vnodeQueryFreeQInfoEx(pQInfo); - + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; if (pColFilter->numOfFilters > 0) { tfree(pColFilter->pFilters); } } - + tfree(pQuery->pFilterInfo); tfree(pQuery->colList); tfree(pQuery->sdata); - + if (pQuery->pSelectExpr != NULL) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SSqlBinaryExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; - + if (pBinExprInfo->numOfCols > 0) { tfree(pBinExprInfo->pReqColumns); tSQLBinaryExprDestroy(&pBinExprInfo->pBinExpr, NULL); } } - + tfree(pQuery->pSelectExpr); } - + if (pQuery->defaultVal != NULL) { tfree(pQuery->defaultVal); } - + tfree(pQuery->pGroupbyExpr); tfree(pQuery); - + dTrace("QInfo:%p QInfo is freed", pQInfo); - + // destroy signature, in order to avoid the query process pass the object safety check memset(pQInfo, 0, sizeof(SQInfo)); tfree(pQInfo); } -static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, - SArray *pTableIdList, void* tsdb, SQInfo **pQInfo) { - int32_t code = TSDB_CODE_SUCCESS; +static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pSqlExprs, pTableIdList); - if ((*pQInfo) == NULL) { - code = TSDB_CODE_SERV_OUT_OF_MEMORY; - goto _error; - } - - SQuery *pQuery = (*pQInfo)->runtimeEnv.pQuery; - dTrace("qmsg:%p create QInfo:%p, QInfo created", pQueryMsg, pQInfo); - - pQuery->window.skey = pQueryMsg->window.skey; - pQuery->window.ekey = pQueryMsg->window.ekey; - pQuery->lastKey = pQuery->window.skey; - - if (sem_init(&(*pQInfo)->dataReady, 0, 0) != 0) { - dError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); - code = TSDB_CODE_APP_ERROR; - goto _error; - } - - vnodeParametersSafetyCheck(pQuery); - - STSBuf *pTSBuf = NULL; - if (pQueryMsg->tsLen > 0) { // open new file to save the result - char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; - pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); - - tsBufResetPos(pTSBuf); - tsBufNextPos(pTSBuf); + /* + * get the file size and set the numOfRows to be the file size, since for tsComp query, + * the returned row size is equalled to 1 + * TODO handle the case that the file is too large to send back one time + */ + if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { + struct stat fstat; + if (stat(pQuery->sdata[0]->data, &fstat) == 0) { + *numOfRows = fstat.st_size; + return fstat.st_size; + } else { + dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); + return 0; + } + } else { + return pQuery->rowSize * (*numOfRows); } +} - // filter the qualified - if ((code = initQInfo(*pQInfo, pTSBuf, tsdb)) != TSDB_CODE_SUCCESS) { - goto _error; +static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { + // the remained number of retrieved rows, not the interpolated result + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + // load data from file to msg buffer + if (isTSCompQuery(pQuery)) { + int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); + + // make sure file exist + if (FD_VALID(fd)) { + size_t s = lseek(fd, 0, SEEK_END); + dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); + + lseek(fd, 0, SEEK_SET); + read(fd, data, s); + close(fd); + + unlink(pQuery->sdata[0]->data); + } else { + dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, + pQuery->sdata[0]->data, strerror(errno)); + } + } else { + doCopyQueryResultToMsg(pQInfo, pQuery->rec.size, data); } - - // if (pQInfo->over == 1) { - // vnodeAddRefCount(pQInfo); // for retrieve procedure - // return pQInfo; - // } - - // dTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, - // pQInfo->refCount); - return code; - -_error: - // table query ref will be decrease during error handling - vnodeFreeQInfo(*pQInfo); - return code; + + pQuery->rec.total += pQuery->rec.size; + dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + + return TSDB_CODE_SUCCESS; + + // todo if interpolation exists, the result may be dump to client by several rounds } int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) { @@ -6057,9 +6047,7 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) SArray *pTableIdList = NULL; SSqlFuncExprMsg** pExprMsg = NULL; wchar_t* tagCond = NULL; - char* nameCond = NULL; - - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &nameCond)) != TSDB_CODE_SUCCESS) { + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond)) != TSDB_CODE_SUCCESS) { return code; } @@ -6088,7 +6076,10 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) // super table query SArray* res = NULL; + bool isSTableQuery = false; if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { + isSTableQuery = true; + STableId* id = taosArrayGet(pTableIdList, 0); id->uid = -1; @@ -6101,8 +6092,13 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) res = pTableIdList; } - code = createQInfo(pQueryMsg, pGroupbyExpr, pExprs, res, tsdb, pQInfo); - + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, res); + if ((*pQInfo) == NULL) { + code = TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + code = initializeQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); + _query_over: if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pTableIdList); @@ -6126,8 +6122,31 @@ _query_over: return TSDB_CODE_SUCCESS; } +void qTableQuery(SQInfo *pQInfo) { + if (pQInfo == NULL || pQInfo->signature != pQInfo) { + dTrace("%p freed abort query", pQInfo); + return; + } + + if (isQueryKilled(pQInfo)) { + dTrace("QInfo:%p it is already killed, abort", pQInfo); + return; + } + + dTrace("QInfo:%p query task is launched", pQInfo); + + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + if (numOfTables == 1) { + singleTableQueryImpl(pQInfo); + } else { + multiTableQueryImpl(pQInfo); + } + + // vnodeDecRefCount(pQInfo); +} + int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo) { - if (pQInfo == NULL || !isQInfoValid(pQInfo)) { + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } @@ -6148,62 +6167,6 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo) { return (pQInfo->code >= 0)? pQInfo->code:(-pQInfo->code); } -static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - - /* - * get the file size and set the numOfRows to be the file size, since for tsComp query, - * the returned row size is equalled to 1 - * TODO handle the case that the file is too large to send back one time - */ - if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { - struct stat fstat; - if (stat(pQuery->sdata[0]->data, &fstat) == 0) { - *numOfRows = fstat.st_size; - return fstat.st_size; - } else { - dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); - return 0; - } - } else { - return pQuery->rowSize * (*numOfRows); - } -} - -static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { - // the remained number of retrieved rows, not the interpolated result - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - // load data from file to msg buffer - if (isTSCompQuery(pQuery)) { - int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); - - // make sure file exist - if (FD_VALID(fd)) { - size_t s = lseek(fd, 0, SEEK_END); - dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); - - lseek(fd, 0, SEEK_SET); - read(fd, data, s); - close(fd); - - unlink(pQuery->sdata[0]->data); - } else { - dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, - pQuery->sdata[0]->data, strerror(errno)); - } - } else { - doCopyQueryResultToMsg(pQInfo, pQuery->rec.size, data); - } - - pQuery->rec.total += pQuery->rec.size; - dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); - - return TSDB_CODE_SUCCESS; - - // todo if interpolation exists, the result may be dump to client by several rounds -} - bool qHasMoreResultsToRetrieve(SQInfo* pQInfo) { if (pQInfo == NULL || pQInfo->signature != pQInfo || pQInfo->code != TSDB_CODE_SUCCESS) { return false; @@ -6222,7 +6185,7 @@ bool qHasMoreResultsToRetrieve(SQInfo* pQInfo) { } int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen) { - if (pQInfo == NULL || !isQInfoValid(pQInfo)) { + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } @@ -6251,7 +6214,7 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { (*pRsp)->completed = 1; // notify no more result to client - vnodeFreeQInfo(pQInfo); + freeQInfo(pQInfo); } return code; diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0ae86007568d3d7a00ba6f7b147ac5f0d00ee74a --- /dev/null +++ b/src/query/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) + +IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) + MESSAGE(STATUS "gTest library found, build unit test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) +ENDIF() \ No newline at end of file diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c23f0f59241d7bbc7527f7fe332fe97b666acbcc --- /dev/null +++ b/src/query/tests/histogramTest.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "tstoken.h" +#include "tutil.h" + +#include "qhistogram.h" + +/* test validate the names for table/database */ +TEST(testCase, histogram_binary_search) { + SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); + + pHisto->numOfEntries = 10; + for (int32_t i = 0; i < 10; ++i) { + pHisto->elems[i].num = 1; + pHisto->elems[i].val = i; + } + + int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1); + assert(idx == 1); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9); + assert(idx == 9); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20); + assert(idx == 10); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1); + assert(idx == 0); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); + assert(idx == 4); + + free(pHisto); +} + +TEST(testCase, histogram_add) { + SHistogramInfo* pHisto = NULL; + + /** + * use arrayList, elapsed time is: + * before: + * 10,000,000 45sec, bin:1000 (-O0) / 17sec. bin:1000, (-O3) + * + * after: + * + */ + struct timeval systemTime; + gettimeofday(&systemTime, NULL); + int64_t st = + (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; + for (int32_t i = 0; i < 10000; ++i) { + tHistogramAdd(&pHisto, i); + // tHistogramPrint(pHisto); + } + // + gettimeofday(&systemTime, NULL); + int64_t et = + (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; + printf("total elapsed time: %ld\n", et - st); + + printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries); + tHistogramPrint(pHisto); + + printf("%ld\n", tHistogramSum(pHisto, 1.5)); + printf("%ld\n", tHistogramSum(pHisto, 2)); + printf("%ld\n", tHistogramSum(pHisto, 3)); + printf("%ld\n", tHistogramSum(pHisto, 4)); + printf("%ld\n", tHistogramSum(pHisto, 5)); + printf("%ld\n", tHistogramSum(pHisto, 6)); + + for (int32_t i = 399; i < 400; ++i) { + printf("val:%d, %ld\n", i, tHistogramSum(pHisto, i)); + } + + double ratio[] = {0 / 100, 20.0 / 100, 88.0 / 100, 100 / 100}; + double* res = tHistogramUniform(pHisto, ratio, 4); + for (int32_t i = 0; i < 4; ++i) { + printf("%f\n", res[i]); + } + + SHistogramInfo* pHisto1 = NULL; + for (int32_t i = (90000 - 1); i >= 80000; --i) { + tHistogramAdd(&pHisto1, i); + } + tHistogramPrint(pHisto1); + + SHistogramInfo* pRes = tHistogramMerge(pHisto1, pHisto, MAX_HISTOGRAM_BIN); + assert(pRes->numOfElems == pHisto->numOfElems + pHisto1->numOfElems); + tHistogramPrint(pRes); + + tHistogramDestroy(&pHisto); + tHistogramDestroy(&pHisto1); + tHistogramDestroy(&pRes); + free(res); +} + +TEST(testCase, heapsort) { + // int32_t num = 20; + // + // SHeapEntry* pEntry = tHeapCreate(num); + // + // for(int32_t i=0; i +#include +#include +#include + +#include "tsqlfunction.h" + +TEST(testCase, patternMatchTest) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + const char* str = "abcdef"; + int32_t ret = patternMatch("a%b%", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tm01"; + ret = patternMatch("tm__", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tkm1"; + ret = patternMatch("t%m1", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tkm1"; + ret = patternMatch("%m1", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = ""; + ret = patternMatch("%_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = "1"; + ret = patternMatch("%__", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = ""; + ret = patternMatch("%", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = " "; + ret = patternMatch("_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "!"; + ret = patternMatch("%_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefg"; + ret = patternMatch("abc%fg", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%fg", str, 7, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%f_", str, 6, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%f_", str, 1, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("ab", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("a%", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("a__", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); +} diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fca9ac8f9d82911975dc9e917b48b93be26142de --- /dev/null +++ b/src/query/tests/resultBufferTest.cpp @@ -0,0 +1,35 @@ +#include +#include +#include + +#include "taos.h" +#include "qresultBuf.h" +#include "tsdb.h" + +namespace { +// simple test +void simpleTest() { + SDiskbasedResultBuf* pResultBuf = NULL; + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64); + + int32_t pageId = 0; + int32_t groupId = 0; + + tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + ASSERT_TRUE(pBufPage != NULL); + + ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64); + ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L); + + SIDList list = getDataBufPagesIdList(pResultBuf, groupId); + ASSERT_EQ(list.size, 1); + + ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); + + destroyResultBuf(pResultBuf); +} +} // namespace + +TEST(testCase, resultBufferTest) { + simpleTest(); +} diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2392bd00b53c52126585d5bf1e795fe8db15f106 --- /dev/null +++ b/src/query/tests/tsBufTest.cpp @@ -0,0 +1,451 @@ +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "tstoken.h" +#include "ttime.h" +#include "tutil.h" +#include "qtsbuf.h" + +namespace { +/** + * + * @param num total number + * @param step gap between two consecutive ts + * @return + */ +int64_t* createTsList(int32_t num, int64_t start, int32_t step) { + int64_t* pList = (int64_t*)malloc(num * sizeof(int64_t)); + + for (int64_t i = 0; i < num; ++i) { + pList[i] = start + i * step; + } + + return pList; +} + +// simple test +void simpleTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + // write 10 ts points + int32_t num = 10; + int64_t tag = 1; + + int64_t* list = createTsList(10, 10000000, 30); + tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); + EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +// one large list of ts, the ts list need to be split into several small blocks +void largeTSTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + // write 10 ts points + int32_t num = 1000000; + int64_t tag = 1; + + int64_t* list = createTsList(num, 10000000, 30); + tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + + // the data has been flush to disk, no data in cache + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void multiTagsTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t tag = 1; + int64_t start = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void multiVnodeTagsTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t start = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + + // 2000 vnodes + for (int32_t j = 0; j < 20; ++j) { + // vnodeId:0 + start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void loadDataTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t oldStart = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + int32_t numOfVnode = 200; + + // 10000 vnodes + for (int32_t j = 0; j < numOfVnode; ++j) { + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %lld\n", i, list[0]); + + free(list); + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + // create from exists file + STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false); + EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder); + EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode); + EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize); + + EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset); + EXPECT_EQ(pNewBuf->pData[0].info.numOfBlocks, pTSBuf->pData[0].info.numOfBlocks); + EXPECT_EQ(pNewBuf->pData[0].info.compLen, pTSBuf->pData[0].info.compLen); + + EXPECT_STREQ(pNewBuf->path, pTSBuf->path); + + tsBufResetPos(pNewBuf); + + int64_t s = taosGetTimestampUs(); + printf("start:%lld\n", s); + + int32_t x = 0; + while (tsBufNextPos(pNewBuf)) { + STSElem elem = tsBufGetElem(pNewBuf); + if (++x == 100000000) { + break; + } + + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + int64_t e = taosGetTimestampUs(); + printf("end:%lld, elapsed:%lld, total obj:%d\n", e, e - s, x); +} + +void randomIncTsTest() {} + +void TSTraverse() { + // 10000 vnodes + int32_t num = 200000; + int64_t oldStart = 10000000; + int32_t numOfTags = 3; + int32_t step = 30; + int32_t numOfVnode = 2; + + STSBuf* pTSBuf = tsBufCreate(true); + + for (int32_t j = 0; j < numOfVnode; ++j) { + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]); + + free(list); + start += step * num; + + list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + tsBufResetPos(pTSBuf); + + //////////////////////////////////////////////////////////////////////////////////////// + // reverse traverse + int64_t s = taosGetTimestampUs(); + printf("start:%lld\n", s); + + pTSBuf->cur.order = TSQL_SO_DESC; + + // complete reverse traverse + int32_t x = 0; + while (tsBufNextPos(pTSBuf)) { + STSElem elem = tsBufGetElem(pTSBuf); + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + // specify the data block with vnode and tags value + tsBufResetPos(pTSBuf); + pTSBuf->cur.order = TSQL_SO_DESC; + + int32_t startVnode = 1; + int32_t startTag = 2; + + tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + + int32_t totalOutput = 10; + while (1) { + STSElem elem = tsBufGetElem(pTSBuf); + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + + if (!tsBufNextPos(pTSBuf)) { + break; + } + + if (--totalOutput <= 0) { + totalOutput = 10; + + tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + + if (startTag == 0) { + startVnode -= 1; + startTag = 3; + } + + if (startVnode < 0) { + break; + } + } + } + + ///////////////////////////////////////////////////////////////////////////////// + // traverse + pTSBuf->cur.order = TSQL_SO_ASC; + tsBufResetPos(pTSBuf); + + // complete forwards traverse + while (tsBufNextPos(pTSBuf)) { + STSElem elem = tsBufGetElem(pTSBuf); + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + // specify the data block with vnode and tags value + tsBufResetPos(pTSBuf); + pTSBuf->cur.order = TSQL_SO_ASC; + + startVnode = 1; + startTag = 2; + + tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + + totalOutput = 10; + while (1) { + STSElem elem = tsBufGetElem(pTSBuf); + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + + if (!tsBufNextPos(pTSBuf)) { + break; + } + + if (--totalOutput <= 0) { + totalOutput = 10; + + tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + + if (startTag < 0) { + startVnode -= 1; + startTag = 3; + } + + if (startVnode < 0) { + break; + } + } + } +} + +void performanceTest() {} + +void emptyTagTest() {} + +void invalidFileTest() { + const char* cmd = "touch /tmp/test"; + + // create empty file + system(cmd); + + STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true); + EXPECT_TRUE(pNewBuf == NULL); + + pNewBuf = tsBufCreateFromFile("/tmp/911", true); + EXPECT_TRUE(pNewBuf == NULL); +} + +void mergeDiffVnodeBufferTest() { + STSBuf* pTSBuf1 = tsBufCreate(true); + STSBuf* pTSBuf2 = tsBufCreate(true); + + int32_t step = 30; + int32_t num = 1000; + int32_t numOfTags = 10; + + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t)); + + free(list); + + start += step * num; + } + + tsBufFlush(pTSBuf2); + + tsBufMerge(pTSBuf1, pTSBuf2, 9); + EXPECT_EQ(pTSBuf1->numOfVnodes, 2); + EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); + + tsBufDisplay(pTSBuf1); + + tsBufDestory(pTSBuf2); + tsBufDestory(pTSBuf1); +} + +void mergeIdenticalVnodeBufferTest() { + STSBuf* pTSBuf1 = tsBufCreate(true); + STSBuf* pTSBuf2 = tsBufCreate(true); + + int32_t step = 30; + int32_t num = 1000; + int32_t numOfTags = 10; + + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + + tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { + int64_t* list = createTsList(num, start, step); + + tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + tsBufFlush(pTSBuf2); + + tsBufMerge(pTSBuf1, pTSBuf2, 12); + EXPECT_EQ(pTSBuf1->numOfVnodes, 1); + EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); + + tsBufResetPos(pTSBuf1); + while (tsBufNextPos(pTSBuf1)) { + STSElem elem = tsBufGetElem(pTSBuf1); + EXPECT_EQ(elem.vnode, 12); + + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + tsBufDestory(pTSBuf1); + tsBufDestory(pTSBuf2); +} +} // namespace + +TEST(testCase, tsBufTest) { + simpleTest(); + largeTSTest(); + multiTagsTest(); + multiVnodeTagsTest(); + loadDataTest(); + invalidFileTest(); + // randomIncTsTest(); + TSTraverse(); + mergeDiffVnodeBufferTest(); + mergeIdenticalVnodeBufferTest(); +} diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c7520c90f9fbdb818d85d8f3b321c49820441f2d --- /dev/null +++ b/src/query/tests/unitTest.cpp @@ -0,0 +1,769 @@ +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "../../client/inc/tscUtil.h" +#include "ttime.h" +#include "tutil.h" +#include "tvariant.h" +#include "ttokendef.h" + +namespace { +int32_t testValidateName(char* name) { + SSQLToken token = {0}; + token.z = name; + token.n = strlen(name); + token.type = 0; + + tSQLGetToken(name, &token.type); + return tscValidateName(&token); +} +} + +static void _init_tvariant_bool(tVariant* t) { + t->i64Key = TSDB_FALSE; + t->nType = TSDB_DATA_TYPE_BOOL; +} + +static void _init_tvariant_tinyint(tVariant* t) { + t->i64Key = -27; + t->nType = TSDB_DATA_TYPE_TINYINT; +} + +static void _init_tvariant_int(tVariant* t) { + t->i64Key = -23997659; + t->nType = TSDB_DATA_TYPE_INT; +} + +static void _init_tvariant_bigint(tVariant* t) { + t->i64Key = -3333333333333; + t->nType = TSDB_DATA_TYPE_BIGINT; +} + +static void _init_tvariant_float(tVariant* t) { + t->dKey = -8991212199.8987878776; + t->nType = TSDB_DATA_TYPE_FLOAT; +} + +static void _init_tvariant_binary(tVariant* t) { + tVariantDestroy(t); + + t->pz = (char*)calloc(1, 20); //"2e3"); + t->nType = TSDB_DATA_TYPE_BINARY; + strcpy(t->pz, "2e5"); + t->nLen = strlen(t->pz); +} + +static void _init_tvariant_nchar(tVariant* t) { + tVariantDestroy(t); + + t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE); + t->nType = TSDB_DATA_TYPE_NCHAR; + wcscpy(t->wpz, L"-2000000.8765"); + t->nLen = wcslen(t->wpz); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +/* test validate the names for table/database */ +TEST(testCase, db_table_name) { + + char t01[] = "abc"; + EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS); + + char t02[] = "'abc'"; + EXPECT_EQ(testValidateName(t02), TSDB_CODE_SUCCESS); + + char t1[] = "abc.def"; + EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS); + printf("%s\n", t1); + + char t2[] = "'abc.def'"; + EXPECT_EQ(testValidateName(t2), TSDB_CODE_SUCCESS); + printf("%s\n", t2); + + char t3[] = "'abc'.def"; + EXPECT_EQ(testValidateName(t3), TSDB_CODE_SUCCESS); + printf("%s\n", t3); + + char t4[] = "'abc'.'def'"; + EXPECT_EQ(testValidateName(t4), TSDB_CODE_SUCCESS); + + char t5[] = "table.'def'"; + EXPECT_EQ(testValidateName(t5), TSDB_CODE_INVALID_SQL); + + char t6[] = "'table'.'def'"; + EXPECT_EQ(testValidateName(t6), TSDB_CODE_INVALID_SQL); + + char t7[] = "'_ab1234'.'def'"; + EXPECT_EQ(testValidateName(t7), TSDB_CODE_SUCCESS); + printf("%s\n", t7); + + char t8[] = "'_ab&^%1234'.'def'"; + EXPECT_EQ(testValidateName(t8), TSDB_CODE_INVALID_SQL); + + char t9[] = "'_123'.'gtest中文'"; + EXPECT_EQ(testValidateName(t9), TSDB_CODE_INVALID_SQL); + + char t10[] = "abc.'gtest中文'"; + EXPECT_EQ(testValidateName(t10), TSDB_CODE_INVALID_SQL); + + char t10_1[] = "abc.'中文gtest'"; + EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_INVALID_SQL); + + char t11[] = "'192.168.0.1'.abc"; + EXPECT_EQ(testValidateName(t11), TSDB_CODE_INVALID_SQL); + + char t12[] = "192.168.0.1.abc"; + EXPECT_EQ(testValidateName(t12), TSDB_CODE_INVALID_SQL); + + char t13[] = "abc."; + EXPECT_EQ(testValidateName(t13), TSDB_CODE_INVALID_SQL); + + char t14[] = ".abc"; + EXPECT_EQ(testValidateName(t14), TSDB_CODE_INVALID_SQL); + + char t15[] = ".'abc'"; + EXPECT_EQ(testValidateName(t15), TSDB_CODE_INVALID_SQL); + + char t16[] = ".abc'"; + EXPECT_EQ(testValidateName(t16), TSDB_CODE_INVALID_SQL); + + char t17[] = "123a.\"abc\""; + EXPECT_EQ(testValidateName(t17), TSDB_CODE_INVALID_SQL); + printf("%s\n", t17); + + char t18[] = "a.\"abc\""; + EXPECT_EQ(testValidateName(t18), TSDB_CODE_SUCCESS); + printf("%s\n", t18); + + char t19[] = "'_ab1234'.'def'.'ab123'"; + EXPECT_EQ(testValidateName(t19), TSDB_CODE_INVALID_SQL); + + char t20[] = "'_ab1234*&^'"; + EXPECT_EQ(testValidateName(t20), TSDB_CODE_INVALID_SQL); + + char t21[] = "'1234_abc'"; + EXPECT_EQ(testValidateName(t21), TSDB_CODE_INVALID_SQL); + + + // =======Containing capital letters================= + char t30[] = "ABC"; + EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS); + + char t31[] = "'ABC'"; + EXPECT_EQ(testValidateName(t31), TSDB_CODE_SUCCESS); + + char t32[] = "ABC.def"; + EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS); + + char t33[] = "'ABC.def"; + EXPECT_EQ(testValidateName(t33), TSDB_CODE_INVALID_SQL); + + char t33_0[] = "abc.DEF'"; + EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_INVALID_SQL); + + char t34[] = "'ABC.def'"; + //int32_t tmp0 = testValidateName(t34); + EXPECT_EQ(testValidateName(t34), TSDB_CODE_SUCCESS); + + char t35[] = "'ABC'.def"; + EXPECT_EQ(testValidateName(t35), TSDB_CODE_SUCCESS); + + char t36[] = "'ABC'.'DEF'"; + EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS); + + char t37[] = "abc.'DEF'"; + EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS); + + char t37_1[] = "abc.'_123DEF'"; + EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS); + + char t38[] = "'abc'.'DEF'"; + EXPECT_EQ(testValidateName(t38), TSDB_CODE_SUCCESS); + + // do not use key words + char t39[] = "table.'DEF'"; + EXPECT_EQ(testValidateName(t39), TSDB_CODE_INVALID_SQL); + + char t40[] = "'table'.'DEF'"; + EXPECT_EQ(testValidateName(t40), TSDB_CODE_INVALID_SQL); + + char t41[] = "'_abXYZ1234'.'deFF'"; + EXPECT_EQ(testValidateName(t41), TSDB_CODE_SUCCESS); + + char t42[] = "'_abDEF&^%1234'.'DIef'"; + EXPECT_EQ(testValidateName(t42), TSDB_CODE_INVALID_SQL); + + char t43[] = "'_123'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t43), TSDB_CODE_INVALID_SQL); + + char t44[] = "'aABC'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t44), TSDB_CODE_INVALID_SQL); + + char t45[] = "'ABC'."; + EXPECT_EQ(testValidateName(t45), TSDB_CODE_INVALID_SQL); + + char t46[] = ".'ABC'"; + EXPECT_EQ(testValidateName(t46), TSDB_CODE_INVALID_SQL); + + char t47[] = "a.\"aTWc\""; + EXPECT_EQ(testValidateName(t47), TSDB_CODE_SUCCESS); + + // ================has space ================= + char t60[] = " ABC "; + EXPECT_EQ(testValidateName(t60), TSDB_CODE_INVALID_SQL); + + char t60_1[] = " ABC "; + EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_INVALID_SQL); + + char t61[] = "' ABC '"; + EXPECT_EQ(testValidateName(t61), TSDB_CODE_SUCCESS); + + char t61_1[] = "' ABC '"; + EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_SUCCESS); + + char t62[] = " ABC . def "; + EXPECT_EQ(testValidateName(t62), TSDB_CODE_INVALID_SQL); + + char t63[] = "' ABC . def "; + EXPECT_EQ(testValidateName(t63), TSDB_CODE_INVALID_SQL); + + char t63_0[] = " abc . DEF ' "; + EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_INVALID_SQL); + + char t64[] = " ' ABC . def ' "; + //int32_t tmp1 = testValidateName(t64); + EXPECT_EQ(testValidateName(t64), TSDB_CODE_INVALID_SQL); + + char t65[] = " ' ABC '. def "; + EXPECT_EQ(testValidateName(t65), TSDB_CODE_INVALID_SQL); + + char t66[] = "' ABC '.' DEF '"; + EXPECT_EQ(testValidateName(t66), TSDB_CODE_SUCCESS); + + char t67[] = "abc . ' DEF '"; + EXPECT_EQ(testValidateName(t67), TSDB_CODE_INVALID_SQL); + + char t68[] = "' abc '.' DEF '"; + EXPECT_EQ(testValidateName(t68), TSDB_CODE_SUCCESS); + + // do not use key words + char t69[] = "table.'DEF'"; + EXPECT_EQ(testValidateName(t69), TSDB_CODE_INVALID_SQL); + + char t70[] = "'table'.'DEF'"; + EXPECT_EQ(testValidateName(t70), TSDB_CODE_INVALID_SQL); + + char t71[] = "'_abXYZ1234 '.' deFF '"; + EXPECT_EQ(testValidateName(t71), TSDB_CODE_SUCCESS); + + char t72[] = "'_abDEF&^%1234'.' DIef'"; + EXPECT_EQ(testValidateName(t72), TSDB_CODE_INVALID_SQL); + + char t73[] = "'_123'.' Gtest中文'"; + EXPECT_EQ(testValidateName(t73), TSDB_CODE_INVALID_SQL); + + char t74[] = "' aABC'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t74), TSDB_CODE_INVALID_SQL); + + char t75[] = "' ABC '."; + EXPECT_EQ(testValidateName(t75), TSDB_CODE_INVALID_SQL); + + char t76[] = ".' ABC'"; + EXPECT_EQ(testValidateName(t76), TSDB_CODE_INVALID_SQL); + + char t77[] = " a . \"aTWc\" "; + EXPECT_EQ(testValidateName(t77), TSDB_CODE_INVALID_SQL); + + char t78[] = " a.\"aTWc \""; + EXPECT_EQ(testValidateName(t78), TSDB_CODE_INVALID_SQL); + + + // ===============muti string by space =================== + // There's no such case. + //char t160[] = "A BC"; + //EXPECT_EQ(testValidateName(t160), TSDB_CODE_INVALID_SQL); + //printf("end:%s\n", t160); + + // There's no such case. + //char t161[] = "' A BC '"; + //EXPECT_EQ(testValidateName(t161), TSDB_CODE_INVALID_SQL); + + char t162[] = " AB C . de f "; + EXPECT_EQ(testValidateName(t162), TSDB_CODE_INVALID_SQL); + + char t163[] = "' AB C . de f "; + EXPECT_EQ(testValidateName(t163), TSDB_CODE_INVALID_SQL); + + char t163_0[] = " ab c . DE F ' "; + EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_INVALID_SQL); + + char t164[] = " ' AB C . de f ' "; + //int32_t tmp2 = testValidateName(t164); + EXPECT_EQ(testValidateName(t164), TSDB_CODE_INVALID_SQL); + + char t165[] = " ' A BC '. de f "; + EXPECT_EQ(testValidateName(t165), TSDB_CODE_INVALID_SQL); + + char t166[] = "' AB C '.' DE F '"; + EXPECT_EQ(testValidateName(t166), TSDB_CODE_INVALID_SQL); + + char t167[] = "ab c . ' D EF '"; + EXPECT_EQ(testValidateName(t167), TSDB_CODE_INVALID_SQL); + + char t168[] = "' a bc '.' DE F '"; + EXPECT_EQ(testValidateName(t168), TSDB_CODE_INVALID_SQL); + +} + +/* test parse time function */ +TEST(testCase, parse_time) { + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + char t1[] = "2018-1-1 1:1:1.952798"; + char t13[] = "1970-1-1 0:0:0"; + + int64_t time = 0, time1 = 0; + + taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); + + char t2[] = "2018-1-1T1:1:1.952Z"; + taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI); + + EXPECT_EQ(time, 1514739661952 + 28800000); + + char t3[] = "2018-1-1 1:01:01.952"; + taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + char t4[] = "2018-1-1 1:01:01.9"; + char t5[] = "2018-1-1 1:01:1.900"; + char t6[] = "2018-01-01 1:1:1.90"; + char t7[] = "2018-01-01 01:01:01.9"; + char t8[] = "2018-01-01 01:01:01.9007865"; + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t9[] = "2017-4-3 1:1:2.980"; + char t10[] = "2017-4-3T2:1:2.98+9:00"; + taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t11[] = "2017-4-3T2:1:2.98+09:00"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t12[] = "2017-4-3T2:1:2.98+0900"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taos_options(TSDB_OPTION_TIMEZONE, "UTC"); + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + char t14[] = "1970-1-1T0:0:0Z"; + taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + char t40[] = "1970-1-1 0:0:0.999999999"; + taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); + + char t41[] = "1997-1-1 0:0:0.999999999"; + taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999); + + int64_t k = timezone; + char t42[] = "1997-1-1T0:0:0.999999999Z"; + taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); + + //////////////////////////////////////////////////////////////////// + // illegal timestamp format + char t15[] = "2017-12-33 0:0:0"; + EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1); + + char t16[] = "2017-12-31 99:0:0"; + EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1); + + char t17[] = "2017-12-31T9:0:0"; + EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1); + + char t18[] = "2017-12-31T9:0:0.Z"; + EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1); + + char t19[] = "2017-12-31 9:0:0.-1"; + EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1); + + char t20[] = "2017-12-31 9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0); + EXPECT_EQ(time, 1514682000100); + + char t21[] = "2017-12-31T9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1); + + char t22[] = "2017-12-31 9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0); + + char t23[] = "2017-12-31T9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0); +} + +TEST(testCase, tvariant_convert) { + // 1. bool data to all other data types + tVariant t = {0}; + _init_tvariant_bool(&t); + + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "FALSE"); + tVariantDestroy(&t); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"FALSE"); + tVariantDestroy(&t); + + // 2. tinyint to other data types + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-27"); + tVariantDestroy(&t); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-27"); + tVariantDestroy(&t); + + // 3. int to other data + // types////////////////////////////////////////////////////////////////// + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + EXPECT_EQ(t.i64Key, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-23997659"); + tVariantDestroy(&t); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-23997659"); + tVariantDestroy(&t); + + // 4. bigint to other data + // type////////////////////////////////////////////////////////////////////////////// + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-3333333333333"); + tVariantDestroy(&t); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-3333333333333"); + tVariantDestroy(&t); + + // 5. float to other data + // types//////////////////////////////////////////////////////////////////////// + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -8991212199); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-8991212199.898788"); + tVariantDestroy(&t); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-8991212199.898788"); + tVariantDestroy(&t); + + // 6. binary to other data types + // ////////////////////////////////////////////////////////////////// + t.pz = "true"; + t.nLen = strlen(t.pz); + t.nType = TSDB_DATA_TYPE_BINARY; + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "2e5"); + tVariantDestroy(&t); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"2e5"); + tVariantDestroy(&t); + + // 7. nchar to other data types + // ////////////////////////////////////////////////////////////////// + t.wpz = L"FALSE"; + t.nLen = wcslen(t.wpz); + t.nType = TSDB_DATA_TYPE_NCHAR; + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_nchar(&t); + EXPECT_LE(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -2000000); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-2000000.8765"); + tVariantDestroy(&t); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-2000000.8765"); + tVariantDestroy(&t); +} + +TEST(testCase, tGetToken_Test) { + char* s = ".123 "; + uint32_t type = 0; + + int32_t len = tSQLGetToken(s, &type); + EXPECT_EQ(type, TK_FLOAT); + EXPECT_EQ(len, strlen(s) - 1); + + char s1[] = "1.123e10 "; + len = tSQLGetToken(s1, &type); + EXPECT_EQ(type, TK_FLOAT); + EXPECT_EQ(len, strlen(s1) - 1); + + char s4[] = "0xff "; + len = tSQLGetToken(s4, &type); + EXPECT_EQ(type, TK_HEX); + EXPECT_EQ(len, strlen(s4) - 1); + + // invalid data type + char s2[] = "e10 "; + len = tSQLGetToken(s2, &type); + EXPECT_FALSE(type == TK_FLOAT); + + char s3[] = "1.1.1.1"; + len = tSQLGetToken(s3, &type); + EXPECT_EQ(type, TK_IPTOKEN); + EXPECT_EQ(len, strlen(s3)); + + char s5[] = "0x "; + len = tSQLGetToken(s5, &type); + EXPECT_FALSE(type == TK_HEX); +} + +static SSQLToken createStrToken(char* s) { + SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; + t.type = TK_STRING; + t.z = s; + t.n = strlen(s); + + return t; +} + +TEST(testCase, isValidNumber_test) { + SSQLToken t1 = createStrToken("123abc"); + + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("0xabc"); + EXPECT_EQ(isValidNumber(&t1), TK_HEX); + + t1 = createStrToken("0b11101"); + EXPECT_EQ(isValidNumber(&t1), TK_BIN); + + t1 = createStrToken(".134abc"); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("1e1 "); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("1+2"); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("-0x123"); + EXPECT_EQ(isValidNumber(&t1), TK_HEX); + + t1 = createStrToken("-1"); + EXPECT_EQ(isValidNumber(&t1), TK_INTEGER); + + t1 = createStrToken("-0b1110"); + EXPECT_EQ(isValidNumber(&t1), TK_BIN); + + t1 = createStrToken("-.234"); + EXPECT_EQ(isValidNumber(&t1), TK_FLOAT); +} + +TEST(testCase, getTempFilePath_test) { + char path[4096] = {0}; + memset(path, 1, 4096); + + getTmpfilePath("new_tmp", path); + printf("%s\n", path); +} + diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 67ecf39f1b2a0564e8f2bcdc0cb39360652a3965..0844146bd4667a58666f4807c98ce8a19884a4bd 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -29,6 +29,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) MESSAGE(STATUS "Failed to find iconv, use default encoding method") ENDIF () ENDIF () + + ADD_SUBDIRECTORY(tests) ELSEIF (TD_WINDOWS_64) ADD_DEFINITIONS(-DUSE_LIBICONV) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) @@ -113,4 +115,3 @@ ENDIF() #ENDIF () - diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..042c925165985a285670f5df845708f92146b353 --- /dev/null +++ b/src/util/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) + +IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) + MESSAGE(STATUS "gTest library found, build unit test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(utilTest tutil gtest pthread) +ENDIF() \ No newline at end of file diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..411c899cc010e3f3ca3c39e1d0ff2c06c1c5e873 --- /dev/null +++ b/src/util/tests/cacheTest.cpp @@ -0,0 +1,142 @@ +#include +#include +#include + +#include "taos.h" +//#include "tsdb.h" + +//#include "testCommon.h" +#include "tstoken.h" +#include "tutil.h" +#include "tcache.h" +#include "ttimer.h" +#include "ttime.h" + +namespace { +int32_t tsMaxMgmtConnections = 10000; +int32_t tsMaxMeterConnections = 200; +} +// test cache +TEST(testCase, client_cache_test) { + const int32_t REFRESH_TIME_IN_SEC = 2; + void* tscTmr = taosTmrInit (tsMaxMgmtConnections*2, 200, 6000, "TSC"); + SCacheObj* tscCacheHandle = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC); + + char* key1 = "test1"; + char* data1 = "test11"; + + char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1), 1); + sleep(REFRESH_TIME_IN_SEC+1); + + printf("obj is still valid: %s\n", cachedObj); + + char* data2 = "test22"; + taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + + /* the object is cleared by cache clean operation */ + cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2), 20); + printf("after updated: %s\n", cachedObj); + + printf("start to remove data from cache\n"); + taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + printf("end of removing data from cache\n"); + + getchar(); + + char* key3 = "test2"; + char* data3 = "kkkkkkk"; + + char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3), 1); + printf("%s\n", cachedObj2); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + + sleep(3); + char* d = (char*) taosCacheAcquireByName(tscCacheHandle, key3); +// assert(d == NULL); + + char* key5 = "test5"; + char* data5 = "data5kkkkk"; + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5), 20); + + char* data6= "new Data after updated"; + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6), 20); + printf("%s\n", cachedObj2); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); + + char* data7 = "add call update procedure"; + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7), 20); + printf("%s\n=======================================\n\n", cachedObj2); + + char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); + taosCacheRelease(tscCacheHandle, (void**) &cc, false); + + char* data8 = "ttft"; + char* key6 = "key6"; + + char* ft = (char*) taosCachePut(tscCacheHandle, key6, data8, strlen(data8), 20); + taosCacheRelease(tscCacheHandle, (void**) &ft, false); + + /** + * 140ns + */ + uint64_t startTime = taosGetTimestampUs(); + printf("Cache Performance Test\nstart time:%lld\n", startTime); + for(int32_t i=0; i<1000; ++i) { + char* dd = (char*) taosCacheAcquireByName(tscCacheHandle, key6); + if (dd != NULL) { +// printf("get the data\n"); + } else { + printf("data has been released\n"); + } + + taosCacheRelease(tscCacheHandle, (void**) &dd, false); + } + + uint64_t endTime = taosGetTimestampUs(); + int64_t el = endTime - startTime; + + printf("End of Test, %lld\nTotal Elapsed Time:%lld us.avg:%f us\n", endTime, el, el/1000.0); + + taosCacheCleanup(tscCacheHandle); +} + +TEST(testCase, cache_resize_test) { + const int32_t REFRESH_TIME_IN_SEC = 2; + void* tscTmr = taosTmrInit (1000*2, 200, 6000, "TSC"); + + auto* pCache = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC); + + char key[256] = {0}; + char data[1024] = "abcdefghijk"; + int32_t len = strlen(data); + + uint64_t startTime = taosGetTimestampUs(); + int32_t num = 10000; + + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "abc_%7d", i); + taosCachePut(pCache, key, data, len, 3600); + } + uint64_t endTime = taosGetTimestampUs(); + + printf("add 10,000,000 object cost:%lld us, avg:%f us\n", endTime - startTime, (endTime-startTime)/(double)num); + + startTime = taosGetTimestampUs(); + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "abc_%7d", i); + void* k = taosCacheAcquireByName(pCache, key); + assert(k != 0); + } + endTime = taosGetTimestampUs(); + printf("retrieve 10,000,000 object cost:%lld us,avg:%f\n", endTime - startTime, (endTime - startTime)/(double)num); + + taosCacheCleanup(pCache); + taosMsleep(20000); + getchar(); +} \ No newline at end of file diff --git a/src/util/tests/hashTest.cpp b/src/util/tests/hashTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..93a19897416aa2bb3e19d1c5e3d3c28da64d8bb6 --- /dev/null +++ b/src/util/tests/hashTest.cpp @@ -0,0 +1,156 @@ +#include +#include +#include +#include + +#include "hash.h" +#include "taos.h" +#include "ttime.h" + +namespace { +// the simple test code for basic operations +void simpleTest() { + auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + // put 400 elements in the hash table + for(int32_t i = -200; i < 200; ++i) { + taosHashPut(hashTable, (const char*) &i, sizeof(int32_t), (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 400); + + for(int32_t i = 0; i < 200; ++i) { + char* p = (char*) taosHashGet(hashTable, (const char*) &i, sizeof(int32_t)); + ASSERT_TRUE(p != nullptr); + ASSERT_EQ(*reinterpret_cast(p), i); + } + + for(int32_t i = 1000; i < 2000; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 400); + + for(int32_t i = 0; i < 100; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 300); + + for(int32_t i = 100; i < 150; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 250); + taosHashCleanup(hashTable); +} + +void stringKeyTest() { + auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + char key[128] = {0}; + + // put 200 elements in the hash table + for(int32_t i = 0; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 1000); + + for(int32_t i = 0; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + char* p = (char*) taosHashGet(hashTable, key, len); + ASSERT_TRUE(p != nullptr); + + ASSERT_EQ(*reinterpret_cast(p), i); + } + + for(int32_t i = 500; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + taosHashRemove(hashTable, key, len); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 500); + + for(int32_t i = 0; i < 499; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + taosHashRemove(hashTable, key, len); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 1); + + taosHashCleanup(hashTable); +} + +void functionTest() { + +} + +/** + * evaluate the performance issue, by add 10million elements in to hash table in + * a single threads situation + */ +void noLockPerformanceTest() { + auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + char key[128] = {0}; + int32_t num = 5000000; + + int64_t st = taosGetTimestampUs(); + + // put 10M elements in the hash table + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), num); + + int64_t et = taosGetTimestampUs(); + printf("Elpased time:%" PRId64 " us to add %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num); + + st = taosGetTimestampUs(); + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + char* p = (char*) taosHashGet(hashTable, key, len); + ASSERT_TRUE(p != nullptr); + + ASSERT_EQ(*reinterpret_cast(p), i); + } + + et = taosGetTimestampUs(); + printf("Elpased time:%" PRId64 " us to fetch all %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num); + + printf("The maximum length of overflow linklist in hash table is:%d\n", taosHashGetMaxOverflowLinkLength(hashTable)); + taosHashCleanup(hashTable); +} + +void multithreadsTest() { + //todo +} + +// check the function robustness +void invalidOperationTest() { + +} + +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, hashTest) { + simpleTest(); + stringKeyTest(); + noLockPerformanceTest(); + multithreadsTest(); +} \ No newline at end of file diff --git a/src/util/tests/stringTest.cpp b/src/util/tests/stringTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b1b06c7f490fbe8dd5adcb0d442f3e1034833bc1 --- /dev/null +++ b/src/util/tests/stringTest.cpp @@ -0,0 +1,178 @@ +#include +#include +#include +#include + +#include "taos.h" +#include "tutil.h" + +TEST(testCase, string_dequote_test) { + char t1[] = "'abc'"; + int32_t len = strdequote(t1); + + EXPECT_EQ(3, len); + EXPECT_STRCASEEQ(t1, "abc"); + + char t2[] = "\"abc\""; + len = strdequote(t2); + + EXPECT_EQ(3, len); + EXPECT_STRCASEEQ(t1, "abc"); + + char t21[] = " abc "; + strtrim(t21); + + EXPECT_STREQ("abc", t21); + EXPECT_EQ(3, strlen(t21)); +} + +TEST(testCase, string_replace_test) { + char t3[] = "abc01abc02abc"; + char* ret = strreplace(t3, "abc", "7"); + + EXPECT_EQ(strlen(ret), 7); + EXPECT_STREQ("7017027", ret); + free(ret); + + char t4[] = "a01a02b03c04d05"; + ret = strreplace(t4, "0", "9999999999"); + + EXPECT_EQ(strlen(ret), 5 * 10 + 10); + EXPECT_STREQ("a99999999991a99999999992b99999999993c99999999994d99999999995", ret); + free(ret); + + char t5[] = "abc"; + ret = strreplace(t5, "abc", "12345678901234567890"); + + EXPECT_EQ(strlen(ret), 20); + EXPECT_STREQ("12345678901234567890", ret); + free(ret); + + char t6[] = "abc"; + ret = strreplace(t6, "def", "abc"); + + EXPECT_EQ(strlen(ret), 3); + EXPECT_STREQ("abc", ret); + free(ret); + + char t7[] = "abcde000000000000001234"; + ret = strreplace(t7, "ab", "0000000"); + + EXPECT_EQ(strlen(ret), 28); + EXPECT_STREQ("0000000cde000000000000001234", ret); + free(ret); + + char t8[] = "abc\ndef"; + char t[] = {10, 0}; + + char f1[] = "\\n"; + int32_t fx = strlen(f1); + ret = strreplace(t8, "\n", "\\n"); + + EXPECT_EQ(strlen(ret), 8); + EXPECT_STREQ("abc\\ndef", ret); + free(ret); + + char t9[] = "abc\\ndef"; + ret = strreplace(t9, "\\n", "\n"); + + EXPECT_EQ(strlen(ret), 7); + EXPECT_STREQ("abc\ndef", ret); + free(ret); + + char t10[] = "abcdef"; + ret = strreplace(t10, "", "0"); + + EXPECT_EQ(strlen(ret), 6); + EXPECT_STREQ("abcdef", ret); + free(ret); +} + +TEST(testCase, string_tolower_test) { + char t[1024] = {1}; + memset(t, 1, tListLen(t)); + + const char* a1 = "ABC"; + strtolower(t, a1); + EXPECT_STREQ(t, "abc"); + + memset(t, 1, tListLen(t)); + const char* a2 = "ABC\'ABC\'D"; + strtolower(t, a2); + EXPECT_STREQ(t, "abc\'ABC\'d"); + + memset(t, 1, tListLen(t)); + const char* a3 = ""; + strtolower(t, a3); + EXPECT_STREQ(t, ""); + + memset(t, 1, tListLen(t)); + const char* a4 = "\"AbcDEF\""; + strtolower(t, a4); + EXPECT_STREQ(t, a4); + + memset(t, 1, tListLen(t)); + const char* a5 = "1234\"AbcDEF\"456"; + strtolower(t, a5); + EXPECT_STREQ(t, a5); + + memset(t, 1, tListLen(t)); + const char* a6 = "1234"; + strtolower(t, a6); + EXPECT_STREQ(t, a6); +} + +TEST(testCase, string_strnchr_test) { + char t[1024] = {0}; + memset(t, 1, tListLen(t)); + + char a1[] = "AB.C"; + EXPECT_TRUE(strnchr(a1, '.', strlen(a1), true) != NULL); + + char a2[] = "abc."; + EXPECT_TRUE(strnchr(a2, '.', strlen(a2), true) != NULL); + + char a8[] = "abc."; + EXPECT_TRUE(strnchr(a8, '.', 1, true) == NULL); + + char a3[] = ".abc"; + EXPECT_TRUE(strnchr(a3, '.', strlen(a3), true) != NULL); + + char a4[] = "'.abc'"; + EXPECT_TRUE(strnchr(a4, '.', strlen(a4), true) == NULL); + + char a5[] = "'.abc.'abc"; + EXPECT_TRUE(strnchr(a5, '.', strlen(a5), true) == NULL); + + char a6[] = "0123456789."; + EXPECT_TRUE(strnchr(a6, '.', strlen(a6), true) != NULL); + + char a7[] = "0123456789."; + EXPECT_TRUE(strnchr(a7, '.', 3, true) == NULL); + + char a9[] = "0123456789."; + EXPECT_TRUE(strnchr(a9, '.', 0, true) == NULL); + + char a10[] = "0123456789'.'"; + EXPECT_TRUE(strnchr(a10, '.', strlen(a10), true) == NULL); +} + +TEST(testCase, cache_resize_test) { + char a11[] = "abc'.'"; + EXPECT_TRUE(strnchr(a11, '.', strlen(a11), false) != NULL); + + char a12[] = "abc'-'"; + EXPECT_TRUE(strnchr(a12, '-', strlen(a12), false) != NULL); + + char a15[] = "abc'-'"; + EXPECT_TRUE(strnchr(a15, '-', strlen(a15), true) == NULL); + + char a13[] = "'-'"; + EXPECT_TRUE(strnchr(a13, '-', strlen(a13), false) != NULL); + + char a14[] = "'-'"; + EXPECT_TRUE(strnchr(a14, '-', strlen(a14), true) == NULL); + + char a16[] = "'-'."; + EXPECT_TRUE(strnchr(a16, '.', strlen(a16), true) != NULL); +} \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 761006a0095db206eb7be93fef456a7de1951df3..1dd1c3b29d1dd1d805169e85c39c04d235ad1016 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -349,10 +349,10 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) { } else { // add non-super table to the array pMeta->tables[pTable->tableId.tid] = pTable; - if (pTable->type == TSDB_CHILD_TABLE) { - // add STABLE to the index + if (pTable->type == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index tsdbAddTableIntoIndex(pMeta, pTable); } + pMeta->nTables++; } diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 3119fdc7ccc9287fea28b4364cad75e7faa08784..7b996c8f7a170fdf278c1cb844ca85686f65fa6a 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -73,7 +73,7 @@ typedef struct SQueryFilesInfo { char dbFilePathPrefix[PATH_MAX]; } SQueryFilesInfo; -typedef struct STableQueryInfo { +typedef struct STableCheckInfo { STableId tableId; TSKEY lastKey; STable * pTableObj; @@ -81,7 +81,8 @@ typedef struct STableQueryInfo { int32_t numOfBlocks; int32_t start; SCompBlock *pBlock; -} STableQueryInfo; + SSkipListIterator* iter; +} STableCheckInfo; typedef struct { SCompBlock *compBlock; @@ -90,14 +91,19 @@ typedef struct { typedef struct STableDataBlockInfoEx { SCompBlockFields pBlock; - STableQueryInfo* pMeterDataInfo; + STableCheckInfo* pMeterDataInfo; int32_t blockIndex; int32_t groupIdx; /* number of group is less than the total number of meters */ } STableDataBlockInfoEx; +enum { + SINGLE_TABLE_MODEL = 1, + MULTI_TABLE_MODEL = 2, +}; + typedef struct STsdbQueryHandle { struct STsdbRepo* pTsdb; - + int8_t model; // access model, single table model or multi-table model SQueryHandlePos cur; // current position SQueryHandlePos start; // the start position, used for secondary/third iteration int32_t unzipBufSize; @@ -120,14 +126,13 @@ typedef struct STsdbQueryHandle { bool locateStart; int32_t realNumOfRows; bool loadDataAfterSeek; // load data after seek. - SArray* pTableQueryInfo; + SArray* pTableCheckInfo; int32_t activeIndex; int32_t tableIndex; bool isFirstSlot; void * qinfo; // query info handle, for debug purpose - SSkipListIterator* memIter; STableDataBlockInfoEx *pDataBlockInfoEx; } STsdbQueryHandle; @@ -271,19 +276,21 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond size_t size = taosArrayGetSize(idList); assert(size >= 1); - pQueryHandle->pTableQueryInfo = taosArrayInit(size, sizeof(STableQueryInfo)); + pQueryHandle->pTableCheckInfo = taosArrayInit(size, sizeof(STableCheckInfo)); for(int32_t i = 0; i < size; ++i) { STableId id = *(STableId*) taosArrayGet(idList, i); - STableQueryInfo info = { + STableCheckInfo info = { .lastKey = pQueryHandle->window.skey, .tableId = id, .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), //todo this may be failed }; - taosArrayPush(pQueryHandle->pTableQueryInfo, &info); + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } + pQueryHandle->model = (size > 1)? MULTI_TABLE_MODEL:SINGLE_TABLE_MODEL; + pQueryHandle->activeIndex = 0; // malloc buffer in order to load data from file @@ -314,11 +321,13 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond return (tsdb_query_handle_t)pQueryHandle; } -bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex); +static bool hasMoreDataInCacheForSingleModel(STsdbQueryHandle* pHandle) { + assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); - STable *pTable = pTableQInfo->pTableObj; + STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + STable *pTable = pTableCheckInfo->pTableObj; + assert(pTable != NULL); // no data in cache, abort if (pTable->mem == NULL && pTable->imem == NULL) { @@ -326,13 +335,49 @@ bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { } // all data in mem are checked already. - if (pTableQInfo->lastKey > pTable->mem->keyLast) { + if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { return false; } return true; } +static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) { + size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo); + assert(numOfTables > 0); + + while(pHandle->activeIndex < numOfTables) { + STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + STable *pTable = pTableCheckInfo->pTableObj; + if (pTable->mem == NULL && pTable->imem == NULL) { + pHandle->activeIndex += 1; // try next table if exits + continue; + } + + // all data in mem are checked already. + if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { + pHandle->activeIndex += 1; // try next table if exits + continue; + } + + return true; + } + + // all tables has checked already + return false; +} + +// handle data in cache situation +bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; + if (pHandle->model == SINGLE_TABLE_MODEL) { + return hasMoreDataInCacheForSingleModel(pHandle); + } else { + return hasMoreDataInCacheForMultiModel(pHandle); + } +} + static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pHandle) { int numOfRows = 0; @@ -370,7 +415,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex); + STableCheckInfo* pTableQInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); STable *pTable = pTableQInfo->pTableObj; TSKEY skey = 0, ekey = 0; @@ -379,11 +424,11 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { if (pTable->mem != NULL) { // create mem table iterator if it is not created yet - if (pHandle->memIter == NULL) { - pHandle->memIter = tSkipListCreateIter(pTable->mem->pData); + if (pTableQInfo->iter == NULL) { + pTableQInfo->iter = tSkipListCreateIter(pTable->mem->pData); } - rows = tsdbReadRowsFromCache(pHandle->memIter, INT64_MAX, 2, &skey, &ekey, pHandle); + rows = tsdbReadRowsFromCache(pTableQInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle); } SDataBlockInfo blockInfo = { diff --git a/src/vnode/tsdb/tests/CMakeLists.txt b/src/vnode/tsdb/tests/CMakeLists.txt index 51c15bce203eb920d7f377d261322047ccd008dc..ee1aaba8cd8c623b29f2d1200c640da2a62dfe6e 100644 --- a/src/vnode/tsdb/tests/CMakeLists.txt +++ b/src/vnode/tsdb/tests/CMakeLists.txt @@ -3,9 +3,4 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) add_executable(tsdbTests ${SOURCE_LIST}) target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb) -add_test( - NAME - unit - COMMAND - ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests -) \ No newline at end of file +add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests) \ No newline at end of file