diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index e9d231171512c8cb9c54b0ababf386bfa821dbf2..6bec41d146ee0ad5d8cf313f9e697768310e97af 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -32,7 +32,7 @@ extern "C" { #define UTIL_TABLE_IS_SUPERTABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE)) #define UTIL_TABLE_IS_NOMRAL_TABLE(metaInfo) (!(UTIL_TABLE_IS_SUPERTABLE(metaInfo))) -#define UTIL_TABLE_CREATE_FROM_STABLE(metaInfo) \ +#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE)) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) @@ -106,7 +106,7 @@ bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryOnMetric(SSqlCmd* pCmd); -bool tscQueryMetricTags(SQueryInfo* pQueryInfo); +bool tscQueryTags(SQueryInfo* pQueryInfo); bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd); void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, @@ -176,8 +176,8 @@ void tscIncStreamExecutionCount(void* pStream); bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); // get starter position of metric query condition (query on tags) in SSqlCmd.payload -SCond* tsGetMetricQueryCondPos(STagCond* pCond, uint64_t tableIndex); -void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str); +SCond* tsGetSTableQueryCondPos(STagCond* pCond, uint64_t tableIndex); +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str); void tscTagCondCopy(STagCond* dest, const STagCond* src); void tscTagCondRelease(STagCond* pCond); @@ -207,7 +207,7 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd); void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* keyStr, uint64_t uid); -int tscGetMetricMeta(SSqlObj* pSql, int32_t clauseIndex); +int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index fdc0ae909519eba49b48257d65515537016c3f0a..9cb43a9f3666f330704d968528986b38246b7351 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -70,8 +70,9 @@ typedef struct STableMeta { typedef struct STableMetaInfo { STableMeta * pTableMeta; // table meta, cached in client side and acquried by name - SSuperTableMeta *pMetricMeta; // metricmeta - +// SSuperTableMeta *pMetricMeta; // metricmeta + SArray* vgroupIdList; + /* * 1. keep the vnode index during the multi-vnode super table projection query * 2. keep the vnode index for multi-vnode insertion diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 2bab6e03fe0ab4fd746d4c246299252b4f85fe44..985a6741f95ded4e3f430e5ee656434d2f4cef42 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -406,11 +406,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pSql->fp == NULL) { - tscError("%p callBack is NULL!!!", pSql); - return; - } - if (pSql->fp == (void *)1) { pSql->fp = NULL; @@ -464,7 +459,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - code = tscGetMetricMeta(pSql, 0); + code = tscGetSTableVgroupInfo(pSql, 0); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; @@ -494,7 +489,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - code = tscGetMetricMeta(pSql, pCmd->clauseIndex); + code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; @@ -519,7 +514,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscTansformSQLFunctionForSTableQuery(pQueryInfo); tscIncStreamExecutionCount(pSql->pStream); } else { - tscTrace("%p get tableMeta/metricMeta successfully", pSql); + tscTrace("%p get tableMeta successfully", pSql); } tscDoQuery(pSql); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 46f3ab6687d8fed3110460a5e66f30d0843e8297..42b04d7a16ed53bec33f395fbe2d05adae6541a6 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -286,6 +286,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { // todo add order support static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { +#if 0 // the result structure has been completed in sql parse, so we // only need to reorganize the results in the column format SSqlCmd * pCmd = &pSql->cmd; @@ -337,6 +338,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { } } +#endif return 0; } @@ -345,7 +347,7 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - +#if 0 SSuperTableMeta *pMetricMeta = tscGetMetaInfo(pQueryInfo, 0)->pMetricMeta; int32_t totalNumOfResults = 1; // count function only produce one result int32_t rowLen = tscGetResRowLength(pQueryInfo); @@ -369,7 +371,8 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { } rowIdx++; } - +#endif + return 0; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 5651e5aa389c20bdf1c5a72554de1eabc6364f23..6d01538d8fef438eb18fe05f5adf6bd313cbd7f3 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1238,8 +1238,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _clean; } - if (pCmd->pDataBlocks->nSize > 0) { - // merge according to vgId + if (pCmd->pDataBlocks->nSize > 0) { // merge according to vgId if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { goto _error_clean; } @@ -1294,12 +1293,7 @@ int tsParseInsertSql(SSqlObj *pSql) { int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion) { int32_t ret = TSDB_CODE_SUCCESS; - - if (NULL == pSql->asyncTblPos) { - tscCleanSqlCmd(&pSql->cmd); - } else { - tscTrace("continue parse sql: %s", pSql->asyncTblPos); - } + tscTrace("continue parse sql: %s", pSql->asyncTblPos); if (tscIsInsertOrImportData(pSql->sqlstr)) { /* diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 6427578e82fd73909e04ab8cd9329efafd014e15..2505d544e481a4ad98d212ba4908dae29add35b6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -209,9 +209,14 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); - assert(pQueryInfo->numOfTables == 0); +// assert(pQueryInfo->numOfTables == 0); - STableMetaInfo* pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); + STableMetaInfo* pTableMetaInfo = NULL; + if (pQueryInfo->numOfTables == 0) { + pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); + } else { + pTableMetaInfo = &pQueryInfo->pTableMetaInfo[0]; + } pCmd->command = pInfo->type; @@ -639,7 +644,7 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { * check invalid SQL: * select tbname, tags_fields from super_table_name interval(1s) */ - if (tscQueryMetricTags(pQueryInfo) && pQueryInfo->intervalTime > 0) { + if (tscQueryTags(pQueryInfo) && pQueryInfo->intervalTime > 0) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -746,7 +751,7 @@ int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlO tscClearMeterMetaInfo(pTableMetaInfo, false); } } else { - assert(pTableMetaInfo->pTableMeta == NULL && pTableMetaInfo->pMetricMeta == NULL); + assert(pTableMetaInfo->pTableMeta == NULL); } tfree(oldName); @@ -1252,7 +1257,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (tscQueryMetricTags(pQueryInfo)) { // local handle the metric tag query + if (tscQueryTags(pQueryInfo)) { // local handle the metric tag query pCmd->count = numOfCols; // the number of meter schema, tricky. pQueryInfo->command = TSDB_SQL_RETRIEVE_TAGS; } @@ -1293,7 +1298,7 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c int16_t functionId = (int16_t)((colIdx >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); if (functionId == TSDB_FUNC_TAGPRJ) { - addRequiredTagColumn(pQueryInfo, colIdx - numOfCols, tableIndex); +// addRequiredTagColumn(pQueryInfo, colIdx - numOfCols, tableIndex); pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; } else { pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; @@ -1396,8 +1401,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum SColumnList ids = {0}; ids.ids[0] = *pIndex; - // tag columns do not add to source list - ids.num = (j >= tscGetNumOfColumns(pTableMeta)) ? 0 : 1; + ids.num = 1; insertResultField(pQueryInfo, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name, pExpr); } @@ -2785,7 +2789,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { +static int32_t tablenameListToString(tSQLExpr* pExpr, SStringBuilder* sb) { tSQLExprList* pList = pExpr->pParam; if (pList->nExpr <= 0) { return TSDB_CODE_INVALID_SQL; @@ -2811,7 +2815,7 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuild return TSDB_CODE_SUCCESS; } -static int32_t tablenameCondToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { +static int32_t tablenameCondToString(tSQLExpr* pExpr, SStringBuilder* sb) { taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); taosStringBuilderAppendString(sb, pExpr->val.pz); @@ -3507,7 +3511,7 @@ int tableNameCompar(const void* lhs, const void* rhs) { return ret > 0 ? 1 : -1; } -static int32_t setTableCondForMetricQuery(SQueryInfo* pQueryInfo, const char* account, tSQLExpr* pExpr, +static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* account, tSQLExpr* pExpr, int16_t tableCondIndex, SStringBuilder* sb) { const char* msg = "table name too long"; @@ -3736,7 +3740,7 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, return ret; } - tsSetMetricQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c); + tsSetSTableQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c); doCompactQueryExpr(pExpr); tSQLExprDestroy(p1); @@ -3752,8 +3756,8 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql return TSDB_CODE_SUCCESS; } - const char* msg = "invalid filter expression"; const char* msg1 = "invalid expression"; + const char* msg2 = "invalid filter expression"; int32_t ret = TSDB_CODE_SUCCESS; @@ -3811,11 +3815,11 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql // 7. query condition for table name pQueryInfo->tagCond.relType = (condExpr.relType == TK_AND) ? TSDB_RELATION_AND : TSDB_RELATION_OR; - ret = setTableCondForMetricQuery(pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb); + ret = setTableCondForSTableQuery(pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb); taosStringBuilderDestroy(&sb); if (!validateFilterExpr(pQueryInfo)) { - return invalidSqlErrMsg(pQueryInfo->msg, msg); + return invalidSqlErrMsg(pQueryInfo->msg, msg2); } doAddJoinTagsColumnsIntoTagList(pQueryInfo, &condExpr); @@ -4240,7 +4244,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo))) { return invalidSqlErrMsg(pQueryInfo->msg, msg4); } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) && - UTIL_TABLE_CREATE_FROM_STABLE(pTableMetaInfo)) { + UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg6); } @@ -4666,14 +4670,13 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* * And then launching multiple async-queries against all qualified virtual nodes, during the first-stage * query operation. */ - int32_t code = tscGetMetricMeta(pSql, clauseIndex); + int32_t code = tscGetSTableVgroupInfo(pSql, clauseIndex); if (code != TSDB_CODE_SUCCESS) { return code; } // No tables included. No results generated. Query results are empty. - SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; - if (pTableMetaInfo->pTableMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfTables == 0) { + if (pTableMetaInfo->pTableMeta == NULL) { tscTrace("%p no table in metricmeta, no output result", pSql); pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; } @@ -5441,8 +5444,9 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); // two table: the first one is for current table, and the secondary is for the super table. - tscAddEmptyMetaInfo(pQueryInfo); - assert(pQueryInfo->numOfTables == 2); + if (pQueryInfo->numOfTables < 2) { + tscAddEmptyMetaInfo(pQueryInfo); + } const int32_t TABLE_INDEX = 0; const int32_t STABLE_INDEX = 1; @@ -5687,6 +5691,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr); + + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + int32_t code = tscGetSTableVgroupInfo(pSql, index); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } // parse the group by clause in the first place if (parseGroupbyClause(pQueryInfo, pQuerySql->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 78d29be1af4e388791396fd17c99c61cb08b4edc..2492352284afd2a571d011a4dffa21ef10b932e5 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -32,7 +32,7 @@ int32_t tscGetNumOfTags(const STableMeta* pTableMeta) { } if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) { - assert(tinfo.numOfTags > 0); + assert(tinfo.numOfTags >= 0); return tinfo.numOfTags; } @@ -51,12 +51,12 @@ int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) { SSchema *tscGetTableSchema(const STableMeta *pTableMeta) { assert(pTableMeta != NULL); - if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - STableMeta* pSTableMeta = pTableMeta->pSTable; - assert (pSTableMeta != NULL); - - return pSTableMeta->schema; - } +// if (pTableMeta->tableType == TSDB_CHILD_TABLE) { +// STableMeta* pSTableMeta = pTableMeta->pSTable; +// assert (pSTableMeta != NULL); +// +// return pSTableMeta->schema; +// } return (SSchema*) pTableMeta->schema; } @@ -72,12 +72,14 @@ SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) { STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); - + +#if 0 if (pTableMeta->tableType == TSDB_CHILD_TABLE) { assert (pTableMeta->pSTable != NULL); return pTableMeta->pSTable->tableInfo; } - +#endif + return pTableMeta->tableInfo; } @@ -130,12 +132,13 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t startCol) assert(pTableMeta != NULL); SSchema* pSchema = pTableMeta->schema; - +#if 0 if (pTableMeta->tableType == TSDB_CHILD_TABLE) { assert (pTableMeta->pSTable != NULL); pSchema = pTableMeta->pSTable->schema; } - +#endif + return &pSchema[startCol]; } diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 0ee8afd53c59969f11bfed85c1c32804e48050f9..1d4ed51f32a68cb39d511f83c5ea1f47a3aeb4eb 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -605,7 +605,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pTableMetaInfo->pMetricMeta->numOfVnodes); + (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * 1); if (*pMemBuffer == NULL) { tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -636,7 +636,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); - for (int32_t i = 0; i < pTableMetaInfo->pMetricMeta->numOfVnodes; ++i) { + size_t numOfSubs = taosArrayGetSize(pTableMetaInfo->vgroupIdList); + for (int32_t i = 0; i < numOfSubs; ++i) { (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index decf9875d28e3483ab4b0f064c5d523a9f1e92a0..111ebc4e8c690cef9c52ea214dacde0cf5d58dfd 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -35,7 +35,6 @@ SRpcIpSet tscDnodeIpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); -void (*tscUpdateVnodeMsg[TSDB_SQL_MAX])(SSqlObj *pSql, char *buf); void tscProcessActivityTimer(void *handle, void *tmrId); int tscKeepConn[TSDB_SQL_MAX] = {0}; TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid); @@ -188,7 +187,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .pCont = pMsg, .contLen = pSql->cmd.payloadLen, .handle = pSql, - .code = 0 + .code = 0 }; rpcSendRequest(pVnodeConn, pSql->ipList, &rpcMsg); } else { @@ -315,7 +314,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { pRes->numOfRows += pMsg->affectedRows; tscTrace("%p cmd:%d code:%d, inserted rows:%d, rsp len:%d", pSql, pCmd->command, pRes->code, - *(int32_t *)pRes->pRsp, pRes->rspLen); + pMsg->affectedRows, pRes->rspLen); } else { tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen); } @@ -325,8 +324,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { - int command = pCmd->command; - void *taosres = tscKeepConn[command] ? pSql : NULL; + void *taosres = tscKeepConn[pCmd->command] ? pSql : NULL; + rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows; + tscTrace("%p Async SQL result:%s res:%p", pSql, tstrerror(pRes->code), taosres); /* @@ -360,7 +360,7 @@ int doProcessSql(SSqlObj *pSql) { pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || pCmd->command == TSDB_SQL_META || - pCmd->command == TSDB_SQL_METRIC) { + pCmd->command == TSDB_SQL_STABLEVGROUP) { tscBuildMsg[pCmd->command](pSql, NULL); } @@ -508,69 +508,35 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_SUCCESS; } -void tscUpdateVnodeInSubmitMsg(SSqlObj *pSql, char *buf) { - //SSubmitMsg *pShellMsg; - //char * pMsg; - //STableMetaInfo * pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); - - //STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; - - //pMsg = buf + tsRpcHeadSize; - - //TODO set iplist - //pShellMsg = (SSubmitMsg *)pMsg; - //pShellMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode); - //tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pSql->index].ip), - // htons(pShellMsg->vnode)); -} - int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSubmitMsg *pShellMsg; - char * pMsg, *pStart; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - pStart = pSql->cmd.payload + tsRpcHeadSize; - pMsg = pStart; + char* pMsg = pSql->cmd.payload + tsRpcHeadSize; + + // NOTE: shell message size should not include SMsgDesc + int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc); SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg; - pMsgDesc->numOfVnodes = htonl(1); //set the number of vnodes + + pMsgDesc->numOfVnodes = htonl(1); //todo set the right number of vnodes pMsg += sizeof(SMsgDesc); - pShellMsg = (SSubmitMsg *)pMsg; + SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg; + pShellMsg->header.vgId = htonl(pTableMeta->vgId); - pShellMsg->header.contLen = htonl(pSql->cmd.payloadLen); + pShellMsg->header.contLen = htonl(size); pShellMsg->length = pShellMsg->header.contLen; pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted - // pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here + // pSql->cmd.payloadLen is set during copying data into paylaod pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; + tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htons(pMsgDesc->numOfVnodes)); -// tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip), -// htons(pShellMsg->vnode)); return TSDB_CODE_SUCCESS; } -void tscUpdateVnodeInQueryMsg(SSqlObj *pSql, char *buf) { - //TODO -// SSqlCmd * pCmd = &pSql->cmd; -// STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); -// -// char * pStart = buf + tsRpcHeadSize; -// SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; -// -// if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // pColumnModel == NULL, query on meter -// STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; -// pQueryMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode); -// } else { // query on metric -// SSuperTableMeta * pMetricMeta = pTableMetaInfo->pMetricMeta; -// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); -// pQueryMsg->vnode = htons(pVnodeSidList->vpeerDesc[pSql->index].vnode); -// } -} - /* * for meter query, simply return the size <= 1k * for metric query, estimate size according to meter tags @@ -588,7 +554,10 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize; } - + + int32_t size = 4096; + +#if 0 SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); @@ -599,45 +568,23 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { if (pQueryInfo->tsBuf != NULL) { size += pQueryInfo->tsBuf->fileSize; } - +#endif + return size; } -static char *doSerializeTableInfo(SSqlObj *pSql, int32_t numOfTables, int32_t vgId, char *pMsg) { +static char *doSerializeTableInfo(SSqlObj *pSql, int32_t vgId, char *pMsg) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; - - tscTrace("%p vgId:%d, query on %d tables", pSql, vgId, numOfTables); - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { -#ifdef _DEBUG_VIEW - tscTrace("%p sid:%d, uid:%" PRIu64, pSql, pTableMetaInfo->pTableMeta->sid, pTableMetaInfo->pTableMeta->uid); -#endif - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->sid = htonl(pTableMeta->sid); - pTableIdInfo->uid = htobe64(pTableMeta->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); - pMsg += sizeof(STableIdInfo); - } else { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); - - for (int32_t i = 0; i < numOfTables; ++i) { - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - STableIdInfo *pQueryMeterInfo = tscGetMeterSidInfo(pVnodeSidList, i); - - pTableIdInfo->sid = htonl(pQueryMeterInfo->sid); - pTableIdInfo->uid = htobe64(pQueryMeterInfo->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pQueryMeterInfo->uid)); - - pMsg += sizeof(STableIdInfo); - -#ifdef _DEBUG_VIEW - tscTrace("%p sid:%d, uid:%" PRId64, pSql, pQueryMeterInfo->sid, pQueryMeterInfo->uid); -#endif - } - } - + tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, vgId, pTableMetaInfo->name, pTableMeta->uid); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->sid = htonl(pTableMeta->sid); + pTableIdInfo->uid = htobe64(pTableMeta->uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); + + pMsg += sizeof(STableIdInfo); return pMsg; } @@ -654,12 +601,21 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; if (pQueryInfo->colList.numOfCols <= 0) { tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); return -1; } + + if (pQueryInfo->intervalTime < 0) { + tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); + return -1; + } + + if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { + tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); + return -1; + } char *pStart = pCmd->payload + tsRpcHeadSize; @@ -677,7 +633,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscError("%p error vnodeIdx:%d", pSql, pTableMetaInfo->vnodeIndex); return -1; } - + + uint32_t vnodeId = 1; + +#if 0 SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); uint32_t vnodeId = pVnodeSidList->vpeerDesc[pVnodeSidList->index].vnode; @@ -686,13 +645,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscError("%p vid:%d,error numOfTables in query message:%d", pSql, vnodeId, numOfTables); return -1; // error } - +#endif + tscTrace("%p query on vid:%d, number of tables:%d", pSql, vnodeId, numOfTables); - pQueryMsg->head.vgId = htons(vnodeId); + pQueryMsg->head.vgId = htonl(vnodeId); + numOfTables = 1; } - pQueryMsg->numOfTables = htonl(numOfTables); - if (pQueryInfo->order.order == TSQL_SO_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->stime); pQueryMsg->window.ekey = htobe64(pQueryInfo->etime); @@ -701,6 +660,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->window.ekey = htobe64(pQueryInfo->stime); } + pQueryMsg->numOfTables = htonl(numOfTables); pQueryMsg->order = htons(pQueryInfo->order.order); pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId); pQueryMsg->interpoType = htons(pQueryInfo->interpoType); @@ -710,36 +670,19 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; - - if (pQueryInfo->intervalTime < 0) { - tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); - return -1; - } - - if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { - tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); - return -1; - } - pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // query on meter - pQueryMsg->tagLength = 0; - } else { // query on super table - pQueryMsg->tagLength = htons(pMetricMeta->tagLen); - } - pQueryMsg->queryType = htons(pQueryInfo->type); pQueryMsg->numOfOutputCols = htons(pQueryInfo->exprsInfo.numOfExprs); - if (pQueryInfo->fieldsInfo.numOfOutputCols < 0) { - tscError("%p illegal value of number of output columns in query msg: %d", pSql, - pQueryInfo->fieldsInfo.numOfOutputCols); + int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutputCols; + if (numOfOutput < 0) { + tscError("%p illegal value of number of output columns in query msg: %d", pSql, numOfOutput); return -1; } // set column list ids - char * pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); + char *pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); SSchema *pSchema = tscGetTableSchema(pTableMeta); for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { @@ -791,7 +734,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { bool hasArithmeticFunction = false; SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; - for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); @@ -847,7 +789,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->colNameLen = htonl(len); // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->head.vgId), pMsg); + pMsg = doSerializeTableInfo(pSql, htons(pQueryMsg->head.vgId), pMsg); SSqlGroupbyExpr *pGroupbyExpr = &pQueryInfo->groupbyExpr; if (pGroupbyExpr->numOfGroupCols != 0) { @@ -905,6 +847,35 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); } + // serialize tag column query condition + if (pQueryInfo->tagCond.numOfTagCond > 0) { + STagCond* pTagCond = &pQueryInfo->tagCond; + + SCond *pCond = tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid); + if (pCond != NULL && pCond->cond != NULL) { + size_t condLen = strlen(pCond->cond) + 1; + + bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); + if (!ret) { + tscError("%p mbs to ucs4 failed:%d", pSql, tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid)); + return 0; + } + + pQueryMsg->tagCondLen = htons(condLen); + pMsg += condLen * TSDB_NCHAR_SIZE; + } + } + + // tbname in/like query expression should be sent to mgmt node + STagCond* pTagCond = &pQueryInfo->tagCond; + if (pTagCond->tbnameCond.cond != NULL) { + size_t s = strlen(pTagCond->tbnameCond.cond); + memcpy(pMsg, pTagCond->tbnameCond.cond, s); + + pQueryMsg->nameCondLen = htons(s); + pMsg += s; + } + msgLen = pMsg - pStart; tscTrace("%p msg built success,len:%d bytes", pSql, msgLen); @@ -1442,11 +1413,15 @@ int tscProcessTagRetrieveRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int32_t numOfRes = 0; +#if 0 if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_TAGPRJ) { numOfRes = pTableMetaInfo->pMetricMeta->numOfTables; } else { numOfRes = 1; // for count function, there is only one output. } + +#endif + return tscLocalResultCommonBuilder(pSql, numOfRes); } @@ -1532,8 +1507,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(STagData); } - msgLen = pMsg - (char*)pInfoMsg; - pCmd->payloadLen = msgLen; + pCmd->payloadLen = pMsg - (char*)pInfoMsg;; pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META; tfree(tmpData); @@ -1607,7 +1581,9 @@ static int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); } -int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { +int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + +#if 0 SSuperTableMetaMsg *pMetaMsg; char * pMsg, *pStart; int msgLen = 0; @@ -1670,13 +1646,13 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // convert to unicode before sending to mnode for metric query int32_t condLen = 0; if (pTagCond->numOfTagCond > 0) { - SCond *pCond = tsGetMetricQueryCondPos(pTagCond, uid); + SCond *pCond = tsGetSTableQueryCondPos(pTagCond, uid); if (pCond != NULL && pCond->cond != NULL) { condLen = strlen(pCond->cond) + 1; bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); if (!ret) { - tscError("%p mbs to ucs4 failed:%s", pSql, tsGetMetricQueryCondPos(pTagCond, uid)); + tscError("%p mbs to ucs4 failed:%s", pSql, tsGetSTableQueryCondPos(pTagCond, uid)); return 0; } } @@ -1748,7 +1724,18 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = msgLen; pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; assert(msgLen + minMsgSize() <= size); +#endif + SSqlCmd *pCmd = &pSql->cmd; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + + SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pCmd->payload; + strncpy(pStableVgroupMsg->tableId, pTableMetaInfo->name, tListLen(pStableVgroupMsg->tableId)); + + pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; + pCmd->payloadLen = sizeof(SCMSTableVgroupMsg); + return TSDB_CODE_SUCCESS; } @@ -1817,14 +1804,13 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->vgId = htonl(pMetaMsg->vgId); pMetaMsg->uid = htobe64(pMetaMsg->uid); pMetaMsg->contLen = htons(pMetaMsg->contLen); + pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if (pMetaMsg->sid < 0 || pMetaMsg->vgId < 0) { tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgId, pMetaMsg->sid); return TSDB_CODE_INVALID_VALUE; } - pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - if (pMetaMsg->numOfTags > TSDB_MAX_TAGS || pMetaMsg->numOfTags < 0) { tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); return TSDB_CODE_INVALID_VALUE; @@ -1848,23 +1834,20 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pSchema++; } -// rsp += numOfTotalCols * sizeof(SSchema); -// -// int32_t tagLen = 0; -// SSchema *pTagsSchema = tscGetTableTagSchema(pMetaMsg); -// -// if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { -// for (int32_t i = 0; i < pMetaMsg->numOfTags; ++i) { -// tagLen += pTagsSchema[i].bytes; -// } -// } -// -// rsp += tagLen; -// int32_t size = (int32_t)(rsp - (char *)pMetaMsg); - size_t size = 0; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); +#if 0 + // if current table is created according to super table, get the table meta of super table + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + char id[TSDB_TABLE_ID_LEN + 1] = {0}; + strncpy(id, pMetaMsg->stableId, TSDB_TABLE_ID_LEN); + + // NOTE: if the table meta of super table is not cached at client side yet, the pSTable is NULL + pTableMeta->pSTable = taosCacheAcquireByName(tscCacheHandle, id); + } +#endif + // todo add one more function: taosAddDataIfNotExists(); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); @@ -1878,6 +1861,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { } free(pTableMeta); + return TSDB_CODE_SUCCESS; } @@ -1991,22 +1975,11 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; } -int tscProcessMetricMetaRsp(SSqlObj *pSql) { - SSuperTableMeta *pMeta; - uint8_t ieType; +int tscProcessSTableVgroupRsp(SSqlObj *pSql) { +#if 0 void ** metricMetaList = NULL; int32_t * sizes = NULL; - - char *rsp = pSql->res.pRsp; - - ieType = *rsp; - if (ieType != TSDB_IE_TYPE_META) { - tscError("invalid ie type:%d", ieType); - return TSDB_CODE_INVALID_IE; - } - - rsp++; - + int32_t num = htons(*(int16_t *)rsp); rsp += sizeof(int16_t); @@ -2090,7 +2063,6 @@ int tscProcessMetricMetaRsp(SSqlObj *pSql) { // release the used metricmeta taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); - pTableMetaInfo->pMetricMeta = (SSuperTableMeta *)taosCachePut(tscCacheHandle, name, (char *)metricMetaList[i], sizes[i], tsMetricMetaKeepTimer); tfree(metricMetaList[i]); @@ -2110,7 +2082,23 @@ _error_clean: free(sizes); free(metricMetaList); - +#endif + + SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pSql->res.pRsp; + pStableVgroup->numOfDnodes = htonl(pStableVgroup->numOfDnodes); + + SSqlObj* pparent = pSql->param; + assert(pparent != NULL); + + SSqlCmd* pCmd = &pparent->cmd; + STableMetaInfo* pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + pInfo->vgroupIdList = taosArrayInit(pStableVgroup->numOfDnodes, sizeof(int32_t)); + + // todo opt performance + for(int32_t i = 0; i < pStableVgroup->numOfDnodes; ++i) { + taosArrayPush(pInfo->vgroupIdList, &pStableVgroup->dnodeIps[i]); + } + return pSql->res.code; } @@ -2236,7 +2224,7 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { if (pTableMetaInfo->pTableMeta) { taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); +// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); } return 0; @@ -2257,7 +2245,7 @@ int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { bool isSuperTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo); taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); +// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); if (isSuperTable) { // if it is a super table, reset whole query cache tscTrace("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); @@ -2343,7 +2331,7 @@ int tscProcessRetrieveRspFromLocal(SSqlObj *pSql) { void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); -static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { +static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { tscError("%p malloc failed for new sqlobj to get meter meta", pSql); @@ -2370,7 +2358,7 @@ static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMet STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - strcpy(pNewMeterMetaInfo->name, pTableMetaInfo->name); + strncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, tListLen(pNewMeterMetaInfo->name)); memcpy(pNew->cmd.payload, pSql->cmd.payload, TSDB_DEFAULT_PAYLOAD_SIZE); // tag information if table does not exists. tscTrace("%p new pSqlObj:%p to get tableMeta", pSql, pNew); @@ -2388,7 +2376,7 @@ static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMet int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { assert(strlen(pTableMetaInfo->name) != 0); - // If this STableMetaInfo owns a metermeta, release it first + // If this STableMetaInfo owns a table meta, release it first if (pTableMetaInfo->pTableMeta != NULL) { taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false); } @@ -2401,12 +2389,8 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { return TSDB_CODE_SUCCESS; } - - /* - * for async insert operation, release data block buffer before issue new object to get metermeta - * because in table meta callback function, the tscParse function will generate the submit data blocks - */ - return doGetMeterMetaFromServer(pSql, pTableMetaInfo); + + return getTableMetaFromMgmt(pSql, pTableMetaInfo); } int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) { @@ -2455,7 +2439,7 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { tscWaitingForCreateTable(pCmd); taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); - code = doGetMeterMetaFromServer(pSql, pTableMetaInfo); // todo ?? + code = getTableMetaFromMgmt(pSql, pTableMetaInfo); // todo ?? } else { tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, tscGetNumOfTags(pTableMetaInfo->pTableMeta), pCmd->numOfCols, pTableMetaInfo->pTableMeta->uid, @@ -2465,30 +2449,34 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { return code; } -int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { +int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { int code = TSDB_CODE_NETWORK_UNAVAIL; SSqlCmd *pCmd = &pSql->cmd; - /* - * the query condition is serialized into pCmd->payload, we need to rebuild key for metricmeta info in cache. - */ - bool required = false; + //the query condition is serialized into pCmd->payload, we need to rebuild key for stable meta info in cache. + bool required = false; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); + if (pQueryInfo->pTableMetaInfo[0]->vgroupIdList != NULL) { + return TSDB_CODE_SUCCESS; + } + +#if 0 + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0}; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pTableMetaInfo->pTableMeta->uid); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); +// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); SSuperTableMeta *ppMeta = (SSuperTableMeta *)taosCacheAcquireByName(tscCacheHandle, tagstr); if (ppMeta == NULL) { required = true; break; } else { - pTableMetaInfo->pMetricMeta = ppMeta; +// pTableMetaInfo->pMetricMeta = ppMeta; } } @@ -2496,12 +2484,13 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { if (!required) { return TSDB_CODE_SUCCESS; } - +#endif + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_METRIC; + pNew->cmd.command = TSDB_SQL_STABLEVGROUP; SQueryInfo *pNewQueryInfo = NULL; if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) { @@ -2538,7 +2527,7 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { // tscFreeSubqueryInfo(pCmd); // } - tscTrace("%p allocate new pSqlObj:%p to get metricMeta", pSql, pNew); + tscTrace("%p allocate new pSqlObj:%p to get stable vgroupInfo", pSql, pNew); pNew->fp = tscTableMetaCallBack; pNew->param = pSql; code = tscProcessSql(pNew); @@ -2575,7 +2564,7 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg; tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg; - tscBuildMsg[TSDB_SQL_METRIC] = tscBuildMetricMetaMsg; + tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; tscBuildMsg[TSDB_SQL_MULTI_META] = tscBuildMultiMeterMetaMsg; tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; @@ -2593,7 +2582,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_CONNECT] = tscProcessConnectRsp; tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; - tscProcessMsgRsp[TSDB_SQL_METRIC] = tscProcessMetricMetaRsp; + tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; @@ -2619,7 +2608,4 @@ void tscInitMsgsFp() { tscKeepConn[TSDB_SQL_SELECT] = 1; tscKeepConn[TSDB_SQL_FETCH] = 1; tscKeepConn[TSDB_SQL_HB] = 1; - - tscUpdateVnodeMsg[TSDB_SQL_SELECT] = tscUpdateVnodeInQueryMsg; - tscUpdateVnodeMsg[TSDB_SQL_INSERT] = tscUpdateVnodeInSubmitMsg; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index d62dac088ba73d3bd96720b25a778c1ec273a97c..e4b07a0f645e09892c720899d9be6518c462f844 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -546,11 +546,11 @@ static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) { * if the global limitation is not reached, and current result has not exhausted, or next more vnodes are * available, goes on */ - if (pMetaInfo->vnodeIndex < pMetaInfo->pMetricMeta->numOfVnodes && pRes1->row < pRes1->numOfRows && - (!tscHasReachLimitation(pQueryInfo1, pRes1))) { - allSubqueryExhausted = false; - break; - } +// if (pMetaInfo->vnodeIndex < pMetaInfo->pMetricMeta->numOfVnodes && pRes1->row < pRes1->numOfRows && +// (!tscHasReachLimitation(pQueryInfo1, pRes1))) { +// allSubqueryExhausted = false; +// break; +// } } hasData = !allSubqueryExhausted; @@ -651,8 +651,7 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) { static void asyncFetchCallback(void *param, TAOS_RES *tres, int numOfRows) { SSqlObj* pSql = (SSqlObj*) tres; - if (numOfRows < 0) { - // set the error code + if (numOfRows < 0) { // set the error code pSql->res.code = -numOfRows; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 0b464c362b03b4f890e36827b46dd72c9a2ea491..4fadad5021b5841f482f365f9a6d07ea6777ad9b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -80,7 +80,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; if (code == 0 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - code = tscGetMetricMeta(pSql, 0); + code = tscGetSTableVgroupInfo(pSql, 0); pSql->res.code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 014b0c5cb7fabf70a7d48918fe600a9936536cfd..616bcbba504df2d949a4cbb3aee7ef84b7f6e082 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -178,11 +178,11 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); int numOfTables = 0; if (!UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { - SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; - for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); - numOfTables += pVnodeSidList->numOfSids; - } +// SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; +// for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { +// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); +// numOfTables += pVnodeSidList->numOfSids; +// } } SSubscriptionProgress* progress = (SSubscriptionProgress*)calloc(numOfTables, sizeof(SSubscriptionProgress)); @@ -197,17 +197,17 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { progress[0].uid = uid; progress[0].key = tscGetSubscriptionProgress(pSub, uid); } else { - SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; - numOfTables = 0; - for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); - for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) { - STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j); - int64_t uid = pTableMetaInfo->uid; - progress[numOfTables].uid = uid; - progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid); - } - } +// SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; +// numOfTables = 0; +// for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { +// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); +// for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) { +// STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j); +// int64_t uid = pTableMetaInfo->uid; +// progress[numOfTables].uid = uid; +// progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid); +// } +// } qsort(progress, numOfTables, sizeof(SSubscriptionProgress), tscCompareSubscriptionProgress); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1df977d99626cec34ac3750eb668244dd06c33d4..b0c7b68ab4cfc263f6728d4ff8e95c224ec988cf 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -476,7 +476,8 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { assert(pQueryInfo->numOfTables == 1); // for projection query, need to try next vnode - int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; +// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; + int32_t totalVnode = 0; if ((++pTableMetaInfo->vnodeIndex) < totalVnode) { tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, pTableMetaInfo->vnodeIndex - 1, pTableMetaInfo->vnodeIndex, totalVnode, pRes->numOfTotal); @@ -541,16 +542,16 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { assert(pQueryInfo->numOfTables == 1); // for projection query, need to try next vnode if current vnode is exhausted - if ((++pTableMetaInfo->vnodeIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) { - pSupporter->pState->numOfCompleted = 0; - pSupporter->pState->numOfTotal = 1; - - pSql->cmd.command = TSDB_SQL_SELECT; - pSql->fp = tscJoinQueryCallback; - tscProcessSql(pSql); - - return; - } +// if ((++pTableMetaInfo->vnodeIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) { +// pSupporter->pState->numOfCompleted = 0; +// pSupporter->pState->numOfTotal = 1; +// +// pSql->cmd.command = TSDB_SQL_SELECT; +// pSql->fp = tscJoinQueryCallback; +// tscProcessSql(pSql); +// +// return; +// } } int32_t numOfTotal = pSupporter->pState->numOfTotal; @@ -608,10 +609,10 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes && - (!tscHasReachLimitation(pQueryInfo, pRes))) { - numOfFetch++; - } +// if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes && +// (!tscHasReachLimitation(pQueryInfo, pRes))) { +// numOfFetch++; +// } } else { if (pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pQueryInfo, pRes))) { numOfFetch++; @@ -788,7 +789,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { * data instead of returning to its invoker */ if (pTableMetaInfo->vnodeIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - assert(pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes); +// assert(pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes); pSupporter->pState->numOfCompleted = 0; // reset the record value pSql->fp = joinRetrieveCallback; // continue retrieve data @@ -1005,11 +1006,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pRes->qhandle = 1; // hack the qhandle check - const uint32_t nBufferSize = (1 << 16); // 64KB + const uint32_t nBufferSize = (1u << 16); // 64KB SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes; + + int32_t numOfSubQueries = taosArrayGetSize(pTableMetaInfo->vgroupIdList); assert(numOfSubQueries > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); @@ -1115,7 +1117,7 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { tfree(trsupport); } -static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); +static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) { // set no disk space error info @@ -1137,7 +1139,7 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES pthread_mutex_unlock(&trsupport->queryMutex); - tscRetrieveFromVnodeCallBack(trsupport, tres, trsupport->pState->code); + tscRetrieveFromDnodeCallBack(trsupport, tres, trsupport->pState->code); } static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -1232,7 +1234,86 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq } } -static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { +static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { + int32_t idx = trsupport->subqueryIndex; + SSqlObj * pPObj = trsupport->pParentSqlObj; + tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; + + SSubqueryState* pState = trsupport->pState; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + + // data in from current vnode is stored in cache and disk + uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; +// tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, +// pSvd->vnode, numOfRowsFromSubquery, idx); + + tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); + +#ifdef _DEBUG_VIEW + printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems); + SSrcColumnInfo colInfo[256] = {0}; + tscGetSrcColumnInfo(colInfo, pQueryInfo); + tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems, + trsupport->localBuffer->numOfElems, colInfo); +#endif + + if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { + tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, + tsAvailTmpDirGB, tsMinimalTmpDirGB); + tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE); + return; + } + + // each result for a vnode is ordered as an independant list, + // then used as an input of loser tree for disk-based merge routine + int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, + pQueryInfo->groupbyExpr.orderType); + if (ret != 0) { // set no disk space error info, and abort retry + return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE); + } + + // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion + // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. + // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. + int32_t numOfTotal = pState->numOfTotal; + + int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); + if (finished < numOfTotal) { + tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); + return tscFreeSubSqlObj(trsupport, pSql); + } + + // all sub-queries are returned, start to local merge process + pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; + + tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, + pState->numOfTotal, pState->numOfRetrievedRows); + + SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); + tscClearInterpInfo(pPQueryInfo); + + tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, + &pPObj->cmd, &pPObj->res); + tscTrace("%p build loser tree completed", pPObj); + + pPObj->res.precision = pSql->res.precision; + pPObj->res.numOfRows = 0; + pPObj->res.row = 0; + + // only free once + tfree(trsupport->pState); + tscFreeSubSqlObj(trsupport, pSql); + + // set the command flag must be after the semaphore been correctly set. + pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; + if (pPObj->res.code == TSDB_CODE_SUCCESS) { + (*pPObj->fp)(pPObj->param, pPObj, 0); + } else { + tscQueueAsyncRes(pPObj); + } +} + +static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { SRetrieveSupport *trsupport = (SRetrieveSupport *)param; int32_t idx = trsupport->subqueryIndex; SSqlObj * pPObj = trsupport->pParentSqlObj; @@ -1260,15 +1341,16 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); - SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; +// SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); +// SVnodeSidList *vnodeInfo = 0; +// SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; if (numOfRows > 0) { assert(pRes->numOfRows == numOfRows); int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, - pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); +// tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, +// pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64, @@ -1294,85 +1376,16 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data, pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); - if (ret < 0) { - // set no disk space error info, and abort retry + if (ret < 0) { // set no disk space error info, and abort retry tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - } else { + } else if (pRes->completed) { + tscAllDataRetrievedFromDnode(trsupport, pSql); + } else { // continue fetch data from dnode pthread_mutex_unlock(&trsupport->queryMutex); - taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); - } - - } else { // all data has been retrieved to client - /* data in from current vnode is stored in cache and disk */ - uint32_t numOfRowsFromVnode = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; - tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, - pSvd->vnode, numOfRowsFromVnode, idx); - - tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); - -#ifdef _DEBUG_VIEW - printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems); - SSrcColumnInfo colInfo[256] = {0}; - tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems, - trsupport->localBuffer->numOfElems, colInfo); -#endif - - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, - tsAvailTmpDirGB, tsMinimalTmpDirGB); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - return; - } - - // each result for a vnode is ordered as an independant list, - // then used as an input of loser tree for disk-based merge routine - int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, - pQueryInfo->groupbyExpr.orderType); - if (ret != 0) { - /* set no disk space error info, and abort retry */ - return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - } - - // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion - // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. - // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. - int32_t numOfTotal = pState->numOfTotal; - - int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); - if (finished < numOfTotal) { - tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); - return tscFreeSubSqlObj(trsupport, pSql); - } - - // all sub-queries are returned, start to local merge process - pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; - - tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, - pState->numOfTotal, pState->numOfRetrievedRows); - - SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); - tscClearInterpInfo(pPQueryInfo); - - tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, - &pPObj->cmd, &pPObj->res); - tscTrace("%p build loser tree completed", pPObj); - - pPObj->res.precision = pSql->res.precision; - pPObj->res.numOfRows = 0; - pPObj->res.row = 0; - - // only free once - tfree(trsupport->pState); - tscFreeSubSqlObj(trsupport, pSql); - - // set the command flag must be after the semaphore been correctly set. - pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - if (pPObj->res.code == TSDB_CODE_SUCCESS) { - (*pPObj->fp)(pPObj->param, pPObj, 0); - } else { - tscQueueAsyncRes(pPObj); + taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } + } else { // all data has been retrieved to client + tscAllDataRetrievedFromDnode(trsupport, pSql); } } @@ -1409,10 +1422,10 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SVnodeSidList *vnodeInfo = NULL; SVnodeDesc * pSvd = NULL; - if (pTableMetaInfo->pMetricMeta != NULL) { - vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); - pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; - } +// if (pTableMetaInfo->pMetricMeta != NULL) { +// vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); +// pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; +// } SSubqueryState* pState = trsupport->pState; assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 && @@ -1432,7 +1445,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { /* * if a query on a vnode is failed, all retrieve operations from vnode that occurs later - * than this one are actually not necessary, we simply call the tscRetrieveFromVnodeCallBack + * than this one are actually not necessary, we simply call the tscRetrieveFromDnodeCallBack * function to abort current and remain retrieve process. * * NOTE: threadsafe is required. @@ -1453,7 +1466,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; } else { SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL && pNewQueryInfo->pTableMetaInfo[0]->pMetricMeta != NULL); +// assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL && pNewQueryInfo->pTableMetaInfo[0]->pMetricMeta != NULL); tscProcessSql(pNew); return; } @@ -1470,7 +1483,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->subqueryIndex, pState->code); } - tscRetrieveFromVnodeCallBack(param, tres, pState->code); + tscRetrieveFromDnodeCallBack(param, tres, pState->code); } else { // success, proceed to retrieve data from dnode if (vnodeInfo != NULL) { tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, @@ -1481,7 +1494,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->subqueryIndex); } - taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); + taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index e33fdc70a49869ac3ca6c3a2b1f0f451588d43c1..ec6881db3f5ae4f1e32dbf790bfb662ae3df84c5 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -53,7 +53,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { const int32_t maxKeySize = TSDB_MAX_TAGS_LEN; // allowed max key size - SCond* cond = tsGetMetricQueryCondPos(pTagCond, uid); + SCond* cond = tsGetSTableQueryCondPos(pTagCond, uid); char join[512] = {0}; if (pTagCond->joinInfo.hasJoin) { @@ -92,7 +92,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { free(tmp); } -SCond* tsGetMetricQueryCondPos(STagCond* pTagCond, uint64_t uid) { +SCond* tsGetSTableQueryCondPos(STagCond* pTagCond, uint64_t uid) { for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { if (uid == pTagCond->cond[i].uid) { return &pTagCond->cond[i]; @@ -102,7 +102,8 @@ SCond* tsGetMetricQueryCondPos(STagCond* pTagCond, uint64_t uid) { return NULL; } -void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) { +// todo refactor by using SArray +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) { size_t len = strlen(str); if (len == 0) { return; @@ -122,7 +123,7 @@ bool tscQueryOnMetric(SSqlCmd* pCmd) { (pCmd->msgType == TSDB_MSG_TYPE_QUERY); } -bool tscQueryMetricTags(SQueryInfo* pQueryInfo) { +bool tscQueryTags(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { if (tscSqlExprGet(pQueryInfo, i)->functionId != TSDB_FUNC_TAGPRJ) { return false; @@ -172,6 +173,7 @@ void tscGetDBInfoFromMeterId(char* tableId, char* db) { } SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx) { +#if 0 if (pMetricmeta == NULL) { tscError("illegal metricmeta"); return 0; @@ -189,6 +191,8 @@ SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx } return (SVnodeSidList*)(pMetricmeta->list[vnodeIdx] + (char*)pMetricmeta); +#endif + } STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) { @@ -221,12 +225,12 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { // for select query super table, the metricmeta can not be null in any cases. if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - assert(pTableMetaInfo->pMetricMeta != NULL); +// assert(pTableMetaInfo->pMetricMeta != NULL); } - if (pTableMetaInfo->pMetricMeta == NULL) { - return false; - } +// if (pTableMetaInfo->pMetricMeta == NULL) { +// return false; +// } if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { return false; @@ -259,12 +263,12 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // only query on tag, not a projection query - if (tscQueryMetricTags(pQueryInfo)) { + if (tscQueryTags(pQueryInfo)) { return false; } // for project query, only the following two function is allowed - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) { @@ -1925,7 +1929,7 @@ STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST } pTableMetaInfo->pTableMeta = pTableMeta; - pTableMetaInfo->pMetricMeta = pMetricMeta; +// pTableMetaInfo->pMetricMeta = pMetricMeta; pTableMetaInfo->numOfTags = numOfTags; if (tags != NULL) { @@ -1975,7 +1979,7 @@ void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) } taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache); +// taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache); } void tscResetForNextRetrieve(SSqlRes* pRes) { @@ -2114,15 +2118,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); STableMeta* pPrevMeterMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); - SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta); +// SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta); - pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, - pTableMetaInfo->tagColumnIndex); +// pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, +// pTableMetaInfo->tagColumnIndex); } assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1); if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - assert(pFinalInfo->pMetricMeta != NULL); +// assert(pFinalInfo->pMetricMeta != NULL); } tscTrace( @@ -2222,13 +2226,13 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) { +// if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) { return false; - } +// } - int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; - return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vnodeIndex < totalVnode - 1); +// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; +// return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && +// (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vnodeIndex < totalVnode - 1); } void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { @@ -2244,7 +2248,8 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes)); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; + int32_t totalVnode = 0; +// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; while (++pTableMetaInfo->vnodeIndex < totalVnode) { tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1f53d50c1f3b4f57571508060db4f19616eb49de --- /dev/null +++ b/src/client/tests/timeParseTest.cpp @@ -0,0 +1,167 @@ +#include +#include +#include + +#include "taos.h" +#include "tstoken.h" +#include "ttime.h" +#include "tutil.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +extern void deltaToUtcInitOnce(); +/* test parse time function */ +TEST(testCase, parse_time) { + + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + deltaToUtcInitOnce(); + + char t1[] = "2018-1-1 1:1:1.952798"; + char t13[] = "1970-1-1 0:0:0"; + + int64_t time = 0, time1 = 0; + + taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); + + char t2[] = "2018-1-1T1:1:1.952Z"; + taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI); + + EXPECT_EQ(time, 1514739661952 + 28800000); + + char t3[] = "2018-1-1 1:01:01.952"; + taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + char t4[] = "2018-1-1 1:01:01.9"; + char t5[] = "2018-1-1 1:01:1.900"; + char t6[] = "2018-01-01 1:1:1.90"; + char t7[] = "2018-01-01 01:01:01.9"; + char t8[] = "2018-01-01 01:01:01.9007865"; + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t9[] = "2017-4-3 1:1:2.980"; + char t10[] = "2017-4-3T2:1:2.98+9:00"; + taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t11[] = "2017-4-3T2:1:2.98+09:00"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t12[] = "2017-4-3T2:1:2.98+0900"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taos_options(TSDB_OPTION_TIMEZONE, "UTC"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + deltaToUtcInitOnce(); + + char t14[] = "1970-1-1T0:0:0Z"; + taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + char t40[] = "1970-1-1 0:0:0.999999999"; + taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); + + char t41[] = "1997-1-1 0:0:0.999999999"; + taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999); + + int64_t k = timezone; + char t42[] = "1997-1-1T0:0:0.999999999Z"; + taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); + + //////////////////////////////////////////////////////////////////// + // illegal timestamp format + char t15[] = "2017-12-33 0:0:0"; + EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1); + + char t16[] = "2017-12-31 99:0:0"; + EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1); + + char t17[] = "2017-12-31T9:0:0"; + EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1); + + char t18[] = "2017-12-31T9:0:0.Z"; + EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1); + + char t19[] = "2017-12-31 9:0:0.-1"; + EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1); + + char t20[] = "2017-12-31 9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0); + EXPECT_EQ(time, 1514682000100); + + char t21[] = "2017-12-31T9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1); + + char t22[] = "2017-12-31 9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0); + + char t23[] = "2017-12-31T9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0); + + + //======================== add some case ============================// + + char b1[] = "9999-12-31 23:59:59.999"; + taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 253402271999999); + + + char b2[] = "2020-01-01 01:01:01.321"; + taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1577811661321); + + taos_options(TSDB_OPTION_TIMEZONE, "America/New_York"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND); +} + + diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 1f46c68abcffa57c246c561b4c64b931d3d5c7ea..4e8afd4f0eec26b42842fd4b4ddfcff05d876a13 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -105,11 +105,36 @@ SDataRow tdDataRowDup(SDataRow row); // ----------------- Data column structure typedef struct SDataCol { - int64_t len; - char data[]; + int8_t type; + int16_t colId; + int bytes; + int len; + int offset; + void * pData; } SDataCol; -void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter); +typedef struct { + int maxRowSize; + int maxCols; // max number of columns + int maxPoints; // max number of points + int numOfPoints; + int numOfCols; // Total number of cols + int sversion; // TODO: set sversion + void * buf; + SDataCol cols[]; +} SDataCols; + +#define keyCol(pCols) (&((pCols)->cols[0])) // Key column +#define dataColsKeyAt(pCols, idx) ((int64_t *)(keyCol(pCols)->pData))[(idx)] +#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0) +#define dataColsKeyLast(pCols) dataColsKeyAt(pCols, (pCols)->numOfPoints - 1) + +SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); +void tdResetDataCols(SDataCols *pCols); +void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); +void tdFreeDataCols(SDataCols *pCols); +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols); +void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); #ifdef __cplusplus } diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 04826e43ac98852495b5951aaa0ba66315bf2480..0496fc6feb313597b343f10e366e838a46f1697f 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -294,14 +294,79 @@ SDataRow tdDataRowDup(SDataRow row) { return trow; } -void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter) { - int row = *iter; +SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { + SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * maxCols); + if (pCols == NULL) return NULL; + + pCols->maxRowSize = maxRowSize; + pCols->maxCols = maxCols; + pCols->maxPoints = maxRows; + + pCols->buf = malloc(maxRowSize * maxRows); + if (pCols->buf == NULL) { + free(pCols); + return NULL; + } + + return pCols; +} +void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { + // assert(schemaNCols(pSchema) <= pCols->numOfCols); + tdResetDataCols(pCols); + pCols->numOfCols = schemaNCols(pSchema); + + pCols->cols[0].pData = pCols->buf; for (int i = 0; i < schemaNCols(pSchema); i++) { - // TODO + if (i > 0) { + pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints; + } + pCols->cols[i].type = colType(schemaColAt(pSchema, i)); + pCols->cols[i].bytes = colBytes(schemaColAt(pSchema, i)); + pCols->cols[i].offset = colOffset(schemaColAt(pSchema, i)); + pCols->cols[i].colId = colColId(schemaColAt(pSchema, i)); } - *iter = row + 1; + return pCols; +} + +void tdFreeDataCols(SDataCols *pCols) { + if (pCols) { + if (pCols->buf) free(pCols->buf); + free(pCols); + } +} + +void tdResetDataCols(SDataCols *pCols) { + pCols->numOfPoints = 0; + for (int i = 0; i < pCols->maxCols; i++) { + pCols->cols[i].len = 0; + } +} + +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) { + TSKEY key = dataRowKey(row); + for (int i = 0; i < pCols->numOfCols; i++) { + SDataCol *pCol = pCols->cols + i; + memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, pCol->offset), pCol->bytes); + pCol->len += pCol->bytes; + } + pCols->numOfPoints++; +} +// Pop pointsToPop points from the SDataCols +void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) { + int pointsLeft = pCols->numOfPoints - pointsToPop; + + for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { + SDataCol *p_col = pCols->cols + iCol; + if (p_col->len > 0) { + p_col->len = TYPE_BYTES[p_col->type] * pointsLeft; + if (pointsLeft > 0) { + memmove((void *)(p_col->pData), (void *)((char *)(p_col->pData) + TYPE_BYTES[p_col->type] * pointsToPop), p_col->len); + } + } + } + pCols->numOfPoints = pointsLeft; } /** diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index a23336630e5ef6314db72c4173029e2853566095..cb25cbfd525623d3832187135beadb3981475af9 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -243,16 +243,6 @@ static void dnodeContinueExecuteQuery(void* pVnode, void* qhandle, SReadMsg *pMs taos_queue queue = dnodeGetVnodeRworker(pVnode); taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); - -// SReadMsg readMsg = { -// .rpcMsg = {0}, -// .pCont = qhandle, -// .contLen = 0, -// .pRpcContext = pMsg->pRpcContext, -// }; -// -// taos_queue queue = dnodeGetVnodeRworker(pVnode); -// taosWriteQitem(queue, TSDB_MSG_TYPE_QUERY, &readMsg); } static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) { @@ -261,7 +251,7 @@ static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) { SQInfo* pQInfo = NULL; if (pMsg->contLen != 0) { void* tsdb = dnodeGetVnodeTsdb(pVnode); - int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, NULL, &pQInfo); + int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->code = code; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 49e498d22fa32babe6af475023ca4372e1a23348..4632c67c3cd5dc65267a151f63dcad40cea24b1f 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -28,83 +28,83 @@ extern "C" { #include "trpc.h" // message type -#define TSDB_MSG_TYPE_REG 1 -#define TSDB_MSG_TYPE_REG_RSP 2 -#define TSDB_MSG_TYPE_SUBMIT 3 -#define TSDB_MSG_TYPE_SUBMIT_RSP 4 -#define TSDB_MSG_TYPE_QUERY 5 -#define TSDB_MSG_TYPE_QUERY_RSP 6 -#define TSDB_MSG_TYPE_RETRIEVE 7 -#define TSDB_MSG_TYPE_RETRIEVE_RSP 8 +#define TSDB_MSG_TYPE_REG 1 +#define TSDB_MSG_TYPE_REG_RSP 2 +#define TSDB_MSG_TYPE_SUBMIT 3 +#define TSDB_MSG_TYPE_SUBMIT_RSP 4 +#define TSDB_MSG_TYPE_QUERY 5 +#define TSDB_MSG_TYPE_QUERY_RSP 6 +#define TSDB_MSG_TYPE_RETRIEVE 7 +#define TSDB_MSG_TYPE_RETRIEVE_RSP 8 // message from mnode to dnode -#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9 +#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9 #define TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP 10 -#define TSDB_MSG_TYPE_MD_DROP_TABLE 11 -#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12 -#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13 -#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14 -#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15 +#define TSDB_MSG_TYPE_MD_DROP_TABLE 11 +#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12 +#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13 +#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14 +#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15 #define TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP 16 -#define TSDB_MSG_TYPE_MD_DROP_VNODE 17 -#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18 -#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19 -#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20 -#define TSDB_MSG_TYPE_MD_DROP_STABLE 21 -#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22 -#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23 +#define TSDB_MSG_TYPE_MD_DROP_VNODE 17 +#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18 +#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19 +#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20 +#define TSDB_MSG_TYPE_MD_DROP_STABLE 21 +#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22 +#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23 #define TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP 24 -#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25 +#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25 #define TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP 26 // message from client to mnode -#define TSDB_MSG_TYPE_CM_CONNECT 31 -#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32 -#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33 -#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34 -#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35 -#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36 -#define TSDB_MSG_TYPE_CM_DROP_ACCT 37 -#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38 -#define TSDB_MSG_TYPE_CM_CREATE_USER 39 -#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40 -#define TSDB_MSG_TYPE_CM_ALTER_USER 41 -#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42 -#define TSDB_MSG_TYPE_CM_DROP_USER 43 -#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44 -#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45 +#define TSDB_MSG_TYPE_CM_CONNECT 31 +#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32 +#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33 +#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34 +#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35 +#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36 +#define TSDB_MSG_TYPE_CM_DROP_ACCT 37 +#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38 +#define TSDB_MSG_TYPE_CM_CREATE_USER 39 +#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40 +#define TSDB_MSG_TYPE_CM_ALTER_USER 41 +#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42 +#define TSDB_MSG_TYPE_CM_DROP_USER 43 +#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44 +#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45 #define TSDB_MSG_TYPE_CM_CREATE_DNODE_RSP 46 -#define TSDB_MSG_TYPE_CM_DROP_DNODE 47 -#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48 -#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE +#define TSDB_MSG_TYPE_CM_DROP_DNODE 47 +#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48 +#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE #define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP -#define TSDB_MSG_TYPE_CM_CREATE_DB 49 -#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50 -#define TSDB_MSG_TYPE_CM_DROP_DB 51 -#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52 -#define TSDB_MSG_TYPE_CM_USE_DB 53 -#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54 -#define TSDB_MSG_TYPE_CM_ALTER_DB 55 -#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56 -#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57 +#define TSDB_MSG_TYPE_CM_CREATE_DB 49 +#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50 +#define TSDB_MSG_TYPE_CM_DROP_DB 51 +#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52 +#define TSDB_MSG_TYPE_CM_USE_DB 53 +#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54 +#define TSDB_MSG_TYPE_CM_ALTER_DB 55 +#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56 +#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57 #define TSDB_MSG_TYPE_CM_CREATE_TABLE_RSP 58 -#define TSDB_MSG_TYPE_CM_DROP_TABLE 59 -#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60 -#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61 -#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 -#define TSDB_MSG_TYPE_CM_TABLE_META 63 -#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 -#define TSDB_MSG_TYPE_CM_STABLE_VGROUP 65 +#define TSDB_MSG_TYPE_CM_DROP_TABLE 59 +#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60 +#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61 +#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 +#define TSDB_MSG_TYPE_CM_TABLE_META 63 +#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 +#define TSDB_MSG_TYPE_CM_STABLE_VGROUP 65 #define TSDB_MSG_TYPE_CM_STABLE_VGROUP_RSP 66 -#define TSDB_MSG_TYPE_CM_TABLES_META 67 -#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 -#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 +#define TSDB_MSG_TYPE_CM_TABLES_META 67 +#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 +#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 #define TSDB_MSG_TYPE_CM_ALTER_STREAM_RSP 70 -#define TSDB_MSG_TYPE_CM_SHOW 71 -#define TSDB_MSG_TYPE_CM_SHOW_RSP 72 -#define TSDB_MSG_TYPE_CM_KILL_QUERY 73 -#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74 -#define TSDB_MSG_TYPE_CM_KILL_STREAM 75 +#define TSDB_MSG_TYPE_CM_SHOW 71 +#define TSDB_MSG_TYPE_CM_SHOW_RSP 72 +#define TSDB_MSG_TYPE_CM_KILL_QUERY 73 +#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74 +#define TSDB_MSG_TYPE_CM_KILL_STREAM 75 #define TSDB_MSG_TYPE_CM_KILL_STREAM_RSP 76 #define TSDB_MSG_TYPE_CM_KILL_CONN 77 #define TSDB_MSG_TYPE_CM_KILL_CONN_RSP 78 @@ -188,15 +188,6 @@ extern char *taosMsg[]; #pragma pack(push, 1) -// typedef struct { -// int32_t vnode; -// int32_t sid; -// int32_t sversion; -// uint64_t uid; -// int16_t numOfRows; -// char payLoad[]; -//} SShellSubmitBlock; - typedef struct { int32_t numOfVnodes; } SMsgDesc; @@ -206,14 +197,6 @@ typedef struct SMsgHead { int32_t vgId; } SMsgHead; -// typedef struct { -// SMsgDesc desc; -// SMsgHead header; -// int16_t import; -// int32_t numOfTables; // total number of sid -// char blks[]; // number of data blocks, each table has at least one data block -//} SShellSubmitMsg; - // Submit message for one table typedef struct SSubmitBlk { int64_t uid; // table unique id @@ -482,14 +465,15 @@ typedef struct { int64_t intervalOffset; // start offset for interval query int64_t slidingTime; // value for sliding window char slidingTimeUnit; // time interval type, for revisement of interval(1d) - int16_t tagLength; // tag length in current query + uint16_t tagCondLen; // tag length in current query + uint16_t nameCondLen; // table name in/like query expression string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; - int16_t orderType; // used in group by xx order by xxx + int16_t orderType; // used in group by xx order by xxx uint64_t groupbyTagIds; int64_t limit; int64_t offset; - int16_t queryType; // denote another query process + uint16_t queryType; // denote another query process int16_t numOfOutputCols; // final output columns numbers int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list @@ -630,14 +614,14 @@ typedef struct { char tableIds[]; } SCMMultiTableInfoMsg; -typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; -} SCMSuperTableInfoMsg; +typedef struct SCMSTableVgroupMsg { + char tableId[TSDB_TABLE_ID_LEN]; +} SCMSTableVgroupMsg; typedef struct { int32_t numOfDnodes; uint32_t dnodeIps[]; -} SCMSuperTableInfoRsp; +} SCMSTableVgroupRspMsg; typedef struct { int16_t elemLen; @@ -683,14 +667,15 @@ typedef struct { } SSuperTableMeta; typedef struct STableMetaMsg { - char tableId[TSDB_TABLE_ID_LEN]; // note: This field must be at the front - int32_t contLen; - uint8_t numOfTags; - uint8_t precision; - uint8_t tableType; - int16_t numOfColumns; - int16_t sversion; - + int32_t contLen; + + char tableId[TSDB_TABLE_ID_LEN]; // table id + char stableId[TSDB_TABLE_ID_LEN]; // stable name if it is created according to super table + uint8_t numOfTags; + uint8_t precision; + uint8_t tableType; + int16_t numOfColumns; + int16_t sversion; int8_t numOfVpeers; SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT]; int32_t sid; diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 55ba2543a3f9db16393837853060c06796562bd9..48abae3e9df6f5a061c5816607bb729f764b44e7 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -732,6 +732,7 @@ static int32_t mgmtDoGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STab pMeta->numOfTags = 0; pMeta->numOfColumns = htons((int16_t)pTable->superTable->numOfColumns); pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); + strncpy(pMeta->stableId, pTable->superTable->info.tableId, tListLen(pMeta->stableId)); } else { pMeta->sversion = htons(pTable->sversion); pMeta->numOfTags = 0; diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index cc8ef8f5d423e6a943c2773a93a9d547fe463d29..a5e79fecf37ae9e8f7608d8016f901b0ab801241 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -216,7 +216,7 @@ void* mgmtGetSuperTable(char *tableId) { } static void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) { - SCMSuperTableInfoRsp *rsp = rpcMallocCont(sizeof(SCMSuperTableInfoRsp) + sizeof(uint32_t) * mgmtGetDnodesNum()); + SCMSTableVgroupRspMsg *rsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * mgmtGetDnodesNum()); rsp->numOfDnodes = htonl(1); rsp->dnodeIps[0] = htonl(inet_addr(tsPrivateIp)); return rsp; @@ -628,14 +628,14 @@ void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable) { } static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { - SCMSuperTableInfoMsg *pInfo = pMsg->pCont; + SCMSTableVgroupMsg *pInfo = pMsg->pCont; STableInfo *pTable = mgmtGetSuperTable(pInfo->tableId); if (pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); return; } - SCMSuperTableInfoRsp *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable); + SCMSTableVgroupRspMsg *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable); if (pRsp != NULL) { int32_t msgLen = sizeof(SSuperTableObj) + htonl(pRsp->numOfDnodes) * sizeof(int32_t); SRpcMsg rpcRsp = {0}; diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index 0e51962f49999ae55c1c18aa5acd9d9baba64be7..a6462f98558d163b05d3b8b1cbc2cad23ae7cae4 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -12,4 +12,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(query ${SRC}) TARGET_LINK_LIBRARIES(query tsdb tutil m rt) -ENDIF () \ No newline at end of file +ENDIF () + +ADD_SUBDIRECTORY(tests) \ No newline at end of file diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 1e89ffb49dbc0667dd25d7d8b32646a155bcb016..d52eb24726891a0beb3e4b3aa9c2a061cf902d80 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include #include "os.h" #include "taosmsg.h" @@ -93,8 +94,7 @@ void tSQLBinaryExprToString(tSQLBinaryExpr *pExpr, char *dst, int32_t *len); void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExprs, void (*fp)(void*)); -void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExprs, struct tSkipList *pSkipList, tQueryResultset *result, - SBinaryFilterSupp *param); +void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, char *, int32_t)); diff --git a/src/query/inc/qsqltype.h b/src/query/inc/qsqltype.h index 5d85ab4b80ebad4534ea6fa09b23d3ed97df04c0..ad9affa1ccdc3a7d729646ba1c0bf67063273ad1 100644 --- a/src/query/inc/qsqltype.h +++ b/src/query/inc/qsqltype.h @@ -54,7 +54,7 @@ enum _sql_type { TSDB_SQL_CONNECT, TSDB_SQL_USE_DB, // 30 TSDB_SQL_META, - TSDB_SQL_METRIC, + TSDB_SQL_STABLEVGROUP, TSDB_SQL_MULTI_META, TSDB_SQL_HB, diff --git a/src/query/inc/qtsbuf.h b/src/query/inc/qtsbuf.h index 1afdb0cd6cb9175b14675d06446afbe3a891df27..8e014e5feb781b2c87209118d1c4c8a7c2d13813 100644 --- a/src/query/inc/qtsbuf.h +++ b/src/query/inc/qtsbuf.h @@ -48,10 +48,10 @@ typedef struct STSElem { } STSElem; typedef struct STSCursor { - int32_t vnodeIndex; - int32_t blockIndex; - int32_t tsIndex; - int32_t order; + int32_t vnodeIndex; + int32_t blockIndex; + int32_t tsIndex; + uint32_t order; } STSCursor; typedef struct STSBlock { diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 16f2006d6f54d1a4a9ad94c04b838a7e8a42ca63..5adce04efa44d7e85bc9ccf482dd21e0deff20b5 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -32,12 +32,6 @@ typedef struct SData { char data[]; } SData; -enum { -// ST_QUERY_KILLED = 0, // query killed - ST_QUERY_PAUSED = 1, // query paused, due to full of the response buffer - ST_QUERY_COMPLETED = 2, // query completed -}; - struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); @@ -60,18 +54,20 @@ typedef struct SWindowStatus { } SWindowStatus; typedef struct SWindowResult { - uint16_t numOfRows; + uint16_t numOfRows; // number of rows of current time window SPosInfo pos; // Position of current result in disk-based output buffer SResultInfo* resultInfo; // For each result column, there is a resultInfo STimeWindow window; // The time window that current result covers. - SWindowStatus status; + SWindowStatus status; // this result status: closed or opened } SWindowResult; typedef struct SResultRec { - int64_t total; - int64_t size; - int64_t capacity; - int32_t threshold; // the threshold size, when the number of rows in result buffer, return to client + int64_t total; // total generated result size in rows + int64_t size; // current result set size in rows + int64_t capacity; // capacity of current result output buffer + + // result size threshold in rows. If the result buffer is larger than this, pause query and return to client + int32_t threshold; } SResultRec; typedef struct SWindowResInfo { @@ -99,7 +95,6 @@ typedef struct SSingleColumnFilterInfo { void* pData; } SSingleColumnFilterInfo; -/* intermediate pos during multimeter query involves interval */ typedef struct STableQueryInfo { int64_t lastKey; STimeWindow win; @@ -107,7 +102,7 @@ typedef struct STableQueryInfo { int16_t queryRangeSet; // denote if the query range is set, only available for interval query int64_t tag; STSCursor cur; - int32_t sid; // for retrieve the page id list + int32_t tid; // for retrieve the page id list SWindowResInfo windowResInfo; } STableQueryInfo; @@ -116,7 +111,6 @@ typedef struct STableDataInfo { int32_t numOfBlocks; int32_t start; // start block index int32_t tableIndex; - void* pMeterObj; int32_t groupIdx; // group id in table list STableQueryInfo* pTableQInfo; } STableDataInfo; @@ -172,7 +166,6 @@ typedef struct SQueryRuntimeEnv { typedef struct SQInfo { void* signature; -// void* param; // pointer to the RpcReadMsg TSKEY startTime; TSKEY elapsedTime; int32_t pointsInterpo; @@ -203,7 +196,7 @@ typedef struct SQInfo { * @param pQInfo * @return */ -int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, void* param, SQInfo** pQInfo); +int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo); /** * query on single table @@ -211,12 +204,6 @@ int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, void* par */ void qTableQuery(SQInfo* pQInfo); -/** - * query on super table - * @param pReadMsg - */ -void qSuperTableQuery(void* pReadMsg); - /** * wait for the query completed, and retrieve final results to client * @param pQInfo diff --git a/src/query/src/qast.c b/src/query/src/qast.c index a0cbf121691c5e341964091038d233b40f1c3c58..c6c65eba27e0e96a7ca895a1cdeaeec6fd302029 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -14,6 +14,9 @@ */ #include "qast.h" +#include +#include +#include "../../client/inc/tschemautil.h" #include "os.h" #include "qsqlparser.h" #include "qsyntaxtreefunction.h" @@ -105,7 +108,7 @@ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols, return NULL; } - size_t nodeSize = sizeof(tSQLSyntaxNode); + size_t nodeSize = sizeof(tSQLSyntaxNode); tSQLSyntaxNode *pNode = NULL; if (pToken->type == TK_ID || pToken->type == TK_TBNAME) { @@ -237,9 +240,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tSQLSyntaxNode } static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { - SSQLToken t0; - - t0 = tStrGetToken(str, i, false, 0, NULL); + SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0) { return NULL; } @@ -341,7 +342,8 @@ void tSQLBinaryExprFromString(tSQLBinaryExpr **pExpr, SSchema *pSchema, int32_t return; } - int32_t pos = 0; + int32_t pos = 0; + tSQLSyntaxNode *pStxNode = createSyntaxTree(pSchema, numOfCols, src, &pos); if (pStxNode != NULL) { assert(pStxNode->nodeType == TSQL_NODE_EXPR); @@ -705,126 +707,120 @@ static bool filterItem(tSQLBinaryExpr *pExpr, const void *pItem, SBinaryFilterSu * @param pSchema tag schemas * @param fp filter callback function */ -static UNUSED_FUNC void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, tQueryResultset *pResult, SBinaryFilterSupp *param) { - int32_t n = 0; - for (int32_t i = 0; i < pResult->num; ++i) { - void *pItem = pResult->pRes[i]; +static void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, SArray *pResult, SBinaryFilterSupp *param) { + size_t size = taosArrayGetSize(pResult); + + SArray* array = taosArrayInit(size, POINTER_BYTES); + for (int32_t i = 0; i < size; ++i) { + void *pItem = taosArrayGetP(pResult, i); if (filterItem(pExpr, pItem, param)) { - pResult->pRes[n++] = pResult->pRes[i]; + taosArrayPush(array, &pItem); } } - - pResult->num = n; + + taosArrayCopy(pResult, array); } -//static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, tQueryResultset *pResult, tSkipList *pSkipList, -// SBinaryFilterSupp *param) { -// int32_t n = 0; -// SSkipListIterator iter = {0}; -// -// int32_t ret = tSkipListIteratorReset(pSkipList, &iter); -// assert(ret == 0); -// -// pResult->pRes = calloc(pSkipList->nSize, POINTER_BYTES); -// -// while (tSkipListIteratorNext(&iter)) { -// tSkipListNode *pNode = tSkipListIteratorGet(&iter); -// if (filterItem(pExpr, pNode, param)) { -// pResult->pRes[n++] = pNode; -// } -// } -// -// pResult->num = n; -//} +static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, SArray *pResult, SSkipList *pSkipList, + SBinaryFilterSupp *param) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + if (filterItem(pExpr, pNode, param)) { + taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); + } + } +} // post-root order traverse syntax tree -//void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, tSkipList *pSkipList, tQueryResultset *result, -// SBinaryFilterSupp *param) { -// if (pExpr == NULL) { -// return; -// } -// -// tSQLSyntaxNode *pLeft = pExpr->pLeft; -// tSQLSyntaxNode *pRight = pExpr->pRight; -// -// // recursive traverse left child branch -// if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { -// uint8_t weight = pLeft->pExpr->filterOnPrimaryKey + pRight->pExpr->filterOnPrimaryKey; -// -// if (weight == 0 && result->num > 0 && pSkipList == NULL) { -// /** -// * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. -// * Since no index presented, the filter operation is done by scan all elements in the result set. -// * -// * if the query is a high selectivity filter, only small portion of meters are retrieved. -// */ -// tSQLBinaryTraverseOnResult(pExpr, result, param); -// } else if (weight == 0) { -// /** -// * apply the hierarchical expression to every node in skiplist for find the qualified nodes -// */ -// assert(result->num == 0); -// tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); -// } else if (weight == 2 || (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_OR)) { -// tQueryResultset rLeft = {0}; -// tQueryResultset rRight = {0}; -// -// tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, &rLeft, param); -// tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, &rRight, param); -// -// if (pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS -// intersect(&rLeft, &rRight, result); -// } else if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or -// merge(&rLeft, &rRight, result); -// } else { -// assert(false); -// } -// -// free(rLeft.pRes); -// free(rRight.pRes); -// } else { -// /* -// * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here -// * -// * first, we filter results based on the skiplist index, which is the initial filter stage, -// * then, we conduct the secondary filter operation based on the result from the initial filter stage. -// */ -// assert(pExpr->nSQLBinaryOptr == TSDB_RELATION_AND); -// -// tSQLBinaryExpr *pFirst = NULL; -// tSQLBinaryExpr *pSecond = NULL; -// if (pLeft->pExpr->filterOnPrimaryKey == 1) { -// pFirst = pLeft->pExpr; -// pSecond = pRight->pExpr; -// } else { -// pFirst = pRight->pExpr; -// pSecond = pLeft->pExpr; -// } -// -// assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); -// -// // we filter the result based on the skiplist index in the first place -// tSQLBinaryExprTraverse(pFirst, pSkipList, result, param); -// -// /* -// * recursively perform the filter operation based on the initial results, -// * So, we do not set the skiplist index as a parameter -// */ -// tSQLBinaryExprTraverse(pSecond, NULL, result, param); -// } -// } else { // column project -// assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); -// -// param->setupInfoFn(pExpr, param->pExtInfo); -// if (pSkipList == NULL) { -// tSQLListTraverseOnResult(pExpr, param->fp, result); -// } else { +void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { + if (pExpr == NULL) { + return; + } + + tSQLSyntaxNode *pLeft = pExpr->pLeft; + tSQLSyntaxNode *pRight = pExpr->pRight; + + // recursive traverse left child branch + if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { + uint8_t weight = pLeft->pExpr->filterOnPrimaryKey + pRight->pExpr->filterOnPrimaryKey; + + if (weight == 0 && taosArrayGetSize(result) > 0 && pSkipList == NULL) { + /** + * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. + * Since no index presented, the filter operation is done by scan all elements in the result set. + * + * if the query is a high selectivity filter, only small portion of meters are retrieved. + */ + tSQLBinaryTraverseOnResult(pExpr, result, param); + } else if (weight == 0) { + /** + * apply the hierarchical expression to every node in skiplist for find the qualified nodes + */ + assert(taosArrayGetSize(result) == 0); + tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); + } else if (weight == 2 || (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_OR)) { + tQueryResultset rLeft = {0}; + tQueryResultset rRight = {0}; + + tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, &rLeft, param); + tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, &rRight, param); + + if (pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS + intersect(&rLeft, &rRight, result); + } else if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or + merge(&rLeft, &rRight, result); + } else { + assert(false); + } + + free(rLeft.pRes); + free(rRight.pRes); + } else { + /* + * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here + * + * first, we filter results based on the skiplist index, which is the initial filter stage, + * then, we conduct the secondary filter operation based on the result from the initial filter stage. + */ + assert(pExpr->nSQLBinaryOptr == TSDB_RELATION_AND); + + tSQLBinaryExpr *pFirst = NULL; + tSQLBinaryExpr *pSecond = NULL; + if (pLeft->pExpr->filterOnPrimaryKey == 1) { + pFirst = pLeft->pExpr; + pSecond = pRight->pExpr; + } else { + pFirst = pRight->pExpr; + pSecond = pLeft->pExpr; + } + + assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); + + // we filter the result based on the skiplist index in the first place + tSQLBinaryExprTraverse(pFirst, pSkipList, result, param); + + /* + * recursively perform the filter operation based on the initial results, + * So, we do not set the skiplist index as a parameter + */ + tSQLBinaryExprTraverse(pSecond, NULL, result, param); + } + } else { // column project + assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); + + param->setupInfoFn(pExpr, param->pExtInfo); + if (pSkipList == NULL) { + tSQLListTraverseOnResult(pExpr, param->fp, result); + } else { // assert(result->num == 0); -//// tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result); -// } -// } -//} +// tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result); + } + } +} void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, char *, int32_t)) { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index e9923c6fd8101fe810f40c14f3a87133856c6c9a..83cbae42664407d1245adfc9740fa31be5af1de0 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -375,29 +375,6 @@ bool doRevisedResultsByLimit(SQInfo *pQInfo) { return false; } -/** - * - * @param pQuery - * @param pDataBlockInfo - * @param forwardStep - * @return TRUE means query not completed, FALSE means query is completed - */ -static bool queryPaused(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, int32_t forwardStep) { - // output buffer is full, pause current query - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - // assert((QUERY_IS_ASC_QUERY(pQuery) && forwardStep + pQuery->pos <= pDataBlockInfo->size) || - // (!QUERY_IS_ASC_QUERY(pQuery) && pQuery->pos - forwardStep + 1 >= 0)); - // - return true; - } - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - return true; - } - - return false; -} - static bool isTopBottomQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; @@ -1690,7 +1667,7 @@ bool notHasQueryTimeRange(SQuery *pQuery) { (pQuery->window.skey == INT64_MAX && pQuery->window.ekey == 0 && (!QUERY_IS_ASC_QUERY(pQuery))); } -bool needSupplementaryScan(SQuery *pQuery) { +static bool needReverseScan(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { @@ -2227,7 +2204,7 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { size_t s = taosArrayGetSize(pQInfo->pTableIdList); num = MAX(s, INITIAL_RESULT_ROWS_VALUE); } else { // for super table query, one page for each subset - // num = pQInfo->pSidSet->numOfSubSet; + num = 1;//pQInfo->pSidSet->numOfSubSet; } assert(num > 0); @@ -2664,7 +2641,6 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfRes = 0; SDataStatis *pStatis = NULL; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &numOfRes, &pRuntimeEnv->windowResInfo, pDataBlock); @@ -2950,7 +2926,7 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) return leftTimestamp > rightTimestamp ? 1 : -1; } -int32_t mergeMetersResultToOneGroups(SQInfo *pQInfo) { +int32_t mergeResultsToGroup(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -2990,7 +2966,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { pQInfo->numOfGroupResultPages = 0; // current results of group has been sent to client, try next group - if (mergeMetersResultToOneGroups(pQInfo) != TSDB_CODE_SUCCESS) { + if (mergeResultsToGroup(pQInfo) != TSDB_CODE_SUCCESS) { return; // failed to save data in the disk } @@ -3071,9 +3047,9 @@ int32_t doMergeMetersResultsToGroupRes(SQInfo *pQInfo, STableDataInfo *pTableDat // todo opt for the case of one table per group int32_t numOfMeters = 0; for (int32_t i = start; i < end; ++i) { - int32_t sid = pTableDataInfo[i].pTableQInfo->sid; + int32_t tid = pTableDataInfo[i].pTableQInfo->tid; - SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, sid); + SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, tid); if (list.size > 0 && pTableDataInfo[i].pTableQInfo->windowResInfo.size > 0) { pTableList[numOfMeters] = &pTableDataInfo[i]; numOfMeters += 1; @@ -3240,10 +3216,9 @@ void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pRes } } -void setMeterDataInfo(STableDataInfo *pTableDataInfo, void *pMeterObj, int32_t meterIdx, int32_t groupId) { - pTableDataInfo->pMeterObj = pMeterObj; +void setTableDataInfo(STableDataInfo *pTableDataInfo, int32_t tableIndex, int32_t groupId) { pTableDataInfo->groupIdx = groupId; - pTableDataInfo->tableIndex = meterIdx; + pTableDataInfo->tableIndex = tableIndex; } static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo *pWindowResInfo, int32_t order) { @@ -3297,7 +3272,7 @@ void disableFunctForTableSuppleScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order pQuery->order.order = pQuery->order.order ^ 1u; } -void disableFunctForSuppleScan(SQInfo *pQInfo, int32_t order) { +void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -3322,7 +3297,7 @@ void disableFunctForSuppleScan(SQInfo *pQInfo, int32_t order) { pQuery->order.order = (pQuery->order.order) ^ 1u; } -void enableFunctForMasterScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { +void enableFuncForForwardScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { @@ -3483,7 +3458,7 @@ static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { SQuery * pQuery = pRuntimeEnv->pQuery; SQueryStatus qStatus = {0}; - if (!needSupplementaryScan(pQuery)) { + if (!needReverseScan(pQuery)) { return; } @@ -3503,7 +3478,7 @@ static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { doScanAllDataBlocks(pRuntimeEnv); queryStatusRestore(pRuntimeEnv, &qStatus); - enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + enableFuncForForwardScan(pRuntimeEnv, pQuery->order.order); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } @@ -3562,7 +3537,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { return toContinue; } -void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { +void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); @@ -3588,10 +3563,7 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { break; } - /* - * set the correct start position, and load the corresponding block in buffer for next - * round scan all data blocks. - */ + // set the correct start position, and load the corresponding block in buffer for next round scan all data blocks. int32_t ret = tsdbDataBlockSeek(pRuntimeEnv->pQueryHandle, pos); status = pQuery->status; @@ -3675,18 +3647,13 @@ static bool hasMainOutput(SQuery *pQuery) { return false; } -STableQueryInfo *createMeterQueryInfo(SQInfo *pQInfo, int32_t sid, TSKEY skey, TSKEY ekey) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - +STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, int32_t tid, STimeWindow win) { STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); - pTableQueryInfo->win = (STimeWindow){ - .skey = skey, - .ekey = ekey, - }; - pTableQueryInfo->lastKey = skey; + pTableQueryInfo->win = win; + pTableQueryInfo->lastKey = win.skey; - pTableQueryInfo->sid = sid; + pTableQueryInfo->tid = tid; pTableQueryInfo->cur.vnodeIndex = -1; initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, 100, 100, TSDB_DATA_TYPE_INT); @@ -3702,7 +3669,7 @@ void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) free(pTableQueryInfo); } -void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo, TSKEY skey, TSKEY ekey) { +void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { if (pTableQueryInfo == NULL) { return; } @@ -3906,7 +3873,7 @@ static int32_t getNumOfSubset(SQInfo *pQInfo) { if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (isIntervalQuery(pQuery))) { totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo); } else { - // totalSubset = pQInfo->pSidSet->numOfSubSet; + totalSubset = 1;//pQInfo->pSidSet->numOfSubSet; } return totalSubset; @@ -3942,19 +3909,18 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; int32_t oldOffset = pQInfo->offset; - assert(0); /* * current output space is not enough to keep all the result data of this group, only copy partial results * to SQuery object's result buffer */ - // if (numOfRowsToCopy > pQuery->pointsToRead - numOfResult) { - // numOfRowsToCopy = pQuery->pointsToRead - numOfResult; - // pQInfo->offset += numOfRowsToCopy; - // } else { - // pQInfo->offset = 0; - // pQInfo->subgroupIdx += 1; - // } + if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { + numOfRowsToCopy = pQuery->rec.capacity - numOfResult; + pQInfo->offset += numOfRowsToCopy; + } else { + pQInfo->offset = 0; + pQInfo->subgroupIdx += 1; + } for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; @@ -3965,13 +3931,12 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde } numOfResult += numOfRowsToCopy; - assert(0); - // if (numOfResult == pQuery->rec.pointsToRead) { - // break; - // } + if (numOfResult == pQuery->rec.capacity) { + break; + } } - dTrace("QInfo:%p copy data to SQuery buf completed", GET_QINFO_ADDR(pQuery)); + dTrace("QInfo:%p copy data to SQuery buf completed", pQInfo); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pQuery, numOfResult); @@ -4014,9 +3979,8 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInf } } -void stableApplyFunctionsOnBlock_(SQInfo *pQInfo, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, +void stableApplyFunctionsOnBlock(SQueryRuntimeEnv* pRuntimeEnv, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray *pDataBlock, __block_search_fn_t searchFn) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; @@ -4169,10 +4133,11 @@ int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage } void vnodePrintQueryStatistics(SQInfo *pQInfo) { +#if 0 SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; -#if 0 + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; if (pRuntimeEnv->pResultBuf == NULL) { pSummary->tmpBufferInDisk = 0; @@ -4213,41 +4178,30 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t initQInfo(SQInfo *pQInfo, void *param, void* tsdb) { +int32_t doInitializeQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t code = TSDB_CODE_SUCCESS; - // only the successful complete requries the sem_post/over = 1 operations. - if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || - (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { - dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, - pQuery->window.ekey, pQuery->order.order); - - sem_post(&pQInfo->dataReady); - setQueryStatus(pQuery, QUERY_COMPLETED); - return TSDB_CODE_SUCCESS; - } - setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQuery, false); // dataInCache requires lastKey value pQuery->lastKey = pQuery->window.skey; - STsdbQueryCond cond = {0}; - - cond.twindow = (STimeWindow){.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; - cond.order = pQuery->order.order; - cond.colList = *pQuery->colList; + STsdbQueryCond cond = { + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = *pQuery->colList, + }; SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); for (int32_t i = 0; i < pQuery->numOfCols; ++i) { taosArrayPush(cols, &pQuery->colList[i]); } - - pQInfo->runtimeEnv.pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); - - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; @@ -4258,15 +4212,34 @@ int32_t initQInfo(SQInfo *pQInfo, void *param, void* tsdb) { } // create runtime environment - code = setupQueryRuntimeEnv(&pQInfo->runtimeEnv, NULL, pQuery->order.order, false); + code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { return code; } - pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, false); - if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, isSTableQuery); + + if (isSTableQuery) { + int32_t rows = getInitialPageNum(pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pQuery->intervalTime == 0) { + int16_t type = TSDB_DATA_TYPE_NULL; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group by columns not tags; + type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + } else { + type = TSDB_DATA_TYPE_INT; // group id + } + + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); + } + + } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); if (code != TSDB_CODE_SUCCESS) { return code; @@ -4363,59 +4336,56 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { } } -static void queryOnDataBlocks(SQInfo *pQInfo, STableDataInfo *pMeterDataInfo) { +static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - - // dTrace("QInfo:%p start to check data blocks in %d files", pQInfo, pVnodeFileInfo->numOfFiles); + + int64_t st = taosGetTimestampMs(); + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + tsdb_query_handle_t *pQueryHandle = pRuntimeEnv->pQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(pQInfo)) { break; } - // prepare the STableDataInfo struct for each table - SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - // SMeterObj * pMeterObj = getMeterObj(pSupporter->pMetersHashTable, blockInfo.sid); - - // pQInfo->pObj = pMeterObj; - // pRuntimeEnv->pMeterObj = pMeterObj; - - STableDataInfo *pTableDataInfo = NULL; - // for (int32_t i = 0; i < pSupporter->pSidSet->numOfTables; ++i) { - // if (pMeterDataInfo[i].pMeterObj == pMeterObj) { - // pTableDataInfo = &pMeterDataInfo[i]; - // break; - // } - // } + STableDataInfo* pTableDataInfo = NULL; + + // todo opt performance + for(int32_t i = 0; i < numOfTables; ++i) { + if (pQInfo->pTableDataInfo[i].pTableQInfo->tid == blockInfo.sid) { + pTableDataInfo = &pQInfo->pTableDataInfo[i]; + break; + } + } - assert(pTableDataInfo != NULL); + assert(pTableDataInfo != NULL && pTableDataInfo->pTableQInfo != NULL); STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; - if (pTableDataInfo->pTableQInfo == NULL) { - // pTableDataInfo->pTableQInfo = createMeterQueryInfo(pQInfo, pMeterObj->sid, pQuery->skey, pQuery->ekey); - } - restoreIntervalQueryRange(pRuntimeEnv, pTableQueryInfo); SDataStatis *pStatis = NULL; SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); TSKEY nextKey = blockInfo.window.ekey; - if (pQuery->intervalTime == 0) { + if (!isIntervalQuery(pQuery)) { setExecutionContext(pQInfo, pTableQueryInfo, pTableDataInfo->tableIndex, pTableDataInfo->groupIdx, nextKey); } else { // interval query setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); int32_t ret = setAdditionalInfo(pQInfo, pTableDataInfo->tableIndex, pTableQueryInfo); + if (ret != TSDB_CODE_SUCCESS) { - // pQInfo->killed = 1; - return; + pQInfo->code = ret; + return taosGetTimestampMs() - st; } } - // stableApplyFunctionsOnBlock_(pSupporter, pTableDataInfo, &blockInfo, pStatis, pDataBlock, searchFn); + stableApplyFunctionsOnBlock(pRuntimeEnv, pTableDataInfo, &blockInfo, pStatis, pDataBlock, binarySearchForKey); } + + int64_t et = taosGetTimestampMs(); + return et - st; } static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *dataInCache, int32_t index, @@ -4499,7 +4469,7 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start pointInterpSupporterSetData(pQInfo, &pointInterpSupporter); pointInterpSupporterDestroy(&pointInterpSupporter); - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); // first/last_row query, do not invoke the finalize for super table query doFinalizeResult(pRuntimeEnv); @@ -4525,9 +4495,10 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start */ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - -#if 0 SQuery* pQuery = pRuntimeEnv->pQuery; + setQueryStatus(pQuery, QUERY_COMPLETED); + +#if 0 // tSidSet *pTableIdList = pSupporter->pSidSet; int32_t vid = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[0]->sid)->vnode; @@ -4679,7 +4650,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { } } - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); pQuery->size = getNumOfResult(pRuntimeEnv); doSkipResults(pRuntimeEnv); @@ -4778,95 +4749,85 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { #endif } -static void doOrderedScan(SQInfo *pQInfo) { - SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; -#if 0 -// if (pQInfo->runtimeEnv. == NULL) { -// pSupporter->pMeterDataInfo = calloc(pSupporter->pSidSet->numOfTables, sizeof(STableDataInfo)); -// } - - STableIdInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo; +static void createTableDataInfo(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - tSidSet* pSidset = pSupporter->pSidSet; - int32_t groupId = 0; + // todo make sure the table are added the reference count to gauranteed that all involved tables are valid + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); - for (int32_t i = 0; i < pSidset->numOfTables; ++i) { // load all meter meta info - SMeterObj *pMeterObj = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[i]->sid); - if (pMeterObj == NULL) { - dError("QInfo:%p failed to find required sid:%d", pQInfo, pMeterSidExtInfo[i]->sid); - continue; + if (pQInfo->pTableDataInfo == NULL) { + pQInfo->pTableDataInfo = (STableDataInfo *)calloc(1, sizeof(STableDataInfo) * numOfTables); + if (pQInfo->pTableDataInfo == NULL) { + dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); + pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; + return; } - if (i >= pSidset->starterPos[groupId + 1]) { - groupId += 1; + int32_t groupId = 0; + for (int32_t i = 0; i < numOfTables; ++i) { // load all meter meta info + STableId *id = taosArrayGet(pQInfo->pTableIdList, i); + STableDataInfo *pInfo = &pQInfo->pTableDataInfo[i]; + + setTableDataInfo(pInfo, i, groupId); + pInfo->pTableQInfo = createTableQueryInfo(&pQInfo->runtimeEnv, id->tid, pQuery->window); } - - STableDataInfo *pOneMeterDataInfo = &pSupporter->pMeterDataInfo[i]; - assert(pOneMeterDataInfo->pMeterObj == NULL); - - setMeterDataInfo(pOneMeterDataInfo, pMeterObj, i, groupId); - pOneMeterDataInfo->pTableQInfo = createMeterQueryInfo(pSupporter, pMeterObj->sid, pQuery->skey, pQuery->ekey); - } - - queryOnDataBlocks(pQInfo, pSupporter->pMeterDataInfo); - if (pQInfo->code != TSDB_CODE_SUCCESS) { - return; } -#endif } -static void setupMeterQueryInfoForSupplementQuery(SQInfo *pQInfo) { +static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - int32_t num = taosHashGetSize(pQInfo->pTableIdList); - for (int32_t i = 0; i < num; ++i) { - // STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; - // changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo, pSupporter->rawSKey, pSupporter->rawEKey); + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + + for (int32_t i = 0; i < numOfTables; ++i) { + STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); } } -static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - - if (!needSupplementaryScan(pQuery)) { - dTrace("QInfo:%p no need to do supplementary scan, query completed", pQInfo); - return; - } - +static void doSaveContext(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); - // disableFunctForSuppleScan(pSupporter, pQuery->order.order); - + disableFuncForReverseScan(pQInfo, pQuery->order.order); + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1u; } - -#if 0 - SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); - setupMeterQueryInfoForSupplementQuery(pSupporter); - - int64_t st = taosGetTimestampMs(); - - doOrderedScan(pQInfo); - int64_t et = taosGetTimestampMs(); - dTrace("QInfo:%p supplementary scan completed, elapsed time: %lldms", pQInfo, et - st); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + prepareQueryInfoForReverseScan(pQInfo); +} + +static void doRestoreContext(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; - /* - * restore the env - * the meter query info is not reset to the original state - */ - SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); - enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; } -#endif + + enableFuncForForwardScan(pRuntimeEnv, pQuery->order.order); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } -static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { +static void doCloseAllTimeWindowAfterScan(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + + if (isIntervalQuery(pQuery)) { + for (int32_t i = 0; i < numOfTables; ++i) { + STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + closeAllTimeWindow(&pTableQueryInfo->windowResInfo); + } + } else { // close results for group result + closeAllTimeWindow(&pQInfo->runtimeEnv.windowResInfo); + } +} + +static void multiTableQueryProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -4875,7 +4836,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { * if the subgroupIdx > 0, the query process must be completed yet, we only need to * copy the data into output buffer */ - if (pQuery->intervalTime > 0) { + if (isIntervalQuery(pQuery)) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW @@ -4894,46 +4855,50 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.size, pQuery->rec.total); return; } -#if 0 - pSupporter->pMeterDataInfo = (STableDataInfo *)calloc(1, sizeof(STableDataInfo) * pSupporter->numOfMeters); - if (pSupporter->pMeterDataInfo == NULL) { - dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); - pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; - return; - } - dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, group:%d", pQInfo, pSupporter->rawSKey, - pSupporter->rawEKey, pQuery->order.order, pSupporter->pSidSet->numOfSubSet); + dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, pQuery->window.skey, + pQuery->window.ekey, pQuery->order.order); - dTrace("QInfo:%p main query scan start", pQInfo); - int64_t st = taosGetTimestampMs(); - doOrderedScan(pQInfo); + // create the query support structures + createTableDataInfo(pQInfo); - int64_t et = taosGetTimestampMs(); - dTrace("QInfo:%p main scan completed, elapsed time: %lldms, supplementary scan start, order:%d", pQInfo, et - st, + // do check all qualified data blocks + int64_t el = queryOnDataBlocks(pQInfo); + dTrace("QInfo:%p forward scan completed, elapsed time: %lldms, reversed scan start, order:%d", pQInfo, el, pQuery->order.order ^ 1u); - if (pQuery->intervalTime > 0) { - for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) { - STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; - closeAllTimeWindow(&pTableQueryInfo->windowResInfo); - } - } else { // close results for group result - closeAllTimeWindow(&pRuntimeEnv->windowResInfo); + // query error occurred or query is killed, abort current execution + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { + dTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); + return; } - doMultiMeterSupplementaryScan(pQInfo); + // close all time window results + doCloseAllTimeWindowAfterScan(pQInfo); - if (isQueryKilled(pQInfo)) { - dTrace("QInfo:%p query killed, abort", pQInfo); + if (needReverseScan(pQuery)) { + doSaveContext(pQInfo); + + el = queryOnDataBlocks(pQInfo); + dTrace("QInfo:%p reversed scan completed, elapsed time: %lldms", pQInfo, el); + + doRestoreContext(pQInfo); + } else { + dTrace("QInfo:%p no need to do reversed scan, query completed", pQInfo); + } + + setQueryStatus(pQuery, QUERY_COMPLETED); + + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { + dTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; } - if (pQuery->intervalTime > 0 || isSumAvgRateQuery(pQuery)) { - assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); + if (isIntervalQuery(pQuery) || isSumAvgRateQuery(pQuery)) { +// assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); - if (mergeMetersResultToOneGroups(pSupporter) == TSDB_CODE_SUCCESS) { - copyResToQueryResultBuf(pSupporter, pQuery); + if (mergeResultsToGroup(pQInfo) == TSDB_CODE_SUCCESS) { + copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); @@ -4944,10 +4909,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { } // handle the limitation of output buffer - pQInfo->size += pQuery->size; - dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->size, pQInfo->size, - pQInfo->pointsReturned); -#endif + dTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); } /* @@ -4960,7 +4922,7 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); if (isQueryKilled(pQInfo)) { @@ -4993,7 +4955,7 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { } while (1) { - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); if (isQueryKilled(pQInfo)) { @@ -5039,7 +5001,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { while (1) { initCtxOutputBuf(pRuntimeEnv); - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return; @@ -5076,7 +5038,7 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { while (1) { tableIntervalProcessImpl(pRuntimeEnv); - if (pQuery->intervalTime > 0) { + if (isIntervalQuery(pQuery)) { pQInfo->subgroupIdx = 0; // always start from 0 pQuery->rec.size = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); @@ -5128,138 +5090,100 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { // pQInfo->size - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); } -void qTableQuery(SQInfo *pQInfo) { - if (pQInfo == NULL || pQInfo->signature != pQInfo) { - dTrace("%p freed abort query", pQInfo); - return; - } - - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - - SQuery *pQuery = pRuntimeEnv->pQuery; - if (isQueryKilled(pQInfo)) { - dTrace("QInfo:%p it is already killed, abort", pQInfo); - return; - } +static void singleTableQueryImpl(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; - dTrace("QInfo:%p query task is launched", pQInfo); - if (vnodeHasRemainResults(pQInfo)) { /* * There are remain results that are not returned due to result interpolation * So, we do keep in this procedure instead of launching retrieve procedure for next results. */ int32_t numOfInterpo = 0; - int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); pQuery->rec.size = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, - (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); - + (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); + doRevisedResultsByLimit(pQInfo); - + pQInfo->pointsInterpo += numOfInterpo; pQuery->rec.size += pQuery->rec.size; - - // dTrace("QInfo:%p %d points returned %d points interpo, totalRead:%d totalInterpo:%d totalReturn:%d", - // pQInfo, pQuery->size, numOfInterpo, pQInfo->size, pQInfo->pointsInterpo, - // pQInfo->pointsReturned); + + dTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } - + // here we have scan all qualified data in both data file and cache if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { // continue to get push data from the group result if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || - (pQuery->intervalTime > 0 && pQuery->rec.total < pQuery->limit.limit)) { + ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { + // todo limit the output for interval query? pQuery->rec.size = 0; pQInfo->subgroupIdx = 0; // always start from 0 - + if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); pQuery->rec.size += pQuery->rec.size; - + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); - + if (pQuery->rec.size > 0) { - // dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d - // totalReturn:%d", - // pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->size, - // pQInfo->size, pQInfo->pointsInterpo, pQInfo->pointsReturned); - + dTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } } } - - // dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, - // pMeterObj->sid, - // pMeterObj->meterId, pQInfo->size); - + + dTrace("QInfo:%p query over, %d rows are returned", pQInfo, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); sem_post(&pQInfo->dataReady); return; } - + // number of points returned during this query pQuery->rec.size = 0; - int64_t st = taosGetTimestampUs(); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (pQuery->intervalTime != 0 || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) - // assert(pQuery->checkBufferInLoop == 0 && pQuery->pointsOffset == pQuery->pointsToRead); tableIntervalProcessor(pQInfo); } else { if (isFixedOutputQuery(pQuery)) { assert(pQuery->checkBufferInLoop == 0); - + tableFixedOutputProcessor(pQInfo); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBufferInLoop == 1); tableMultiOutputProcessor(pQInfo); } } - + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); - + /* check if query is killed or not */ if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed", pQInfo); } else { dTrace("QInfo:%p query task completed, %d points are returned", pQInfo, pQuery->rec.size); } - + sem_post(&pQInfo->dataReady); - // vnodeDecRefCount(pQInfo); } -void qSuperTableQuery(void *pReadMsg) { -// SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; -// -// if (pQInfo == NULL) { -// return; -// } - -// if (pQInfo->killed) { -// vnodeDecRefCount(pQInfo); -// dTrace("QInfo:%p it is already killed, abort", pQInfo); -// return; -// } - -// assert(pQInfo->refCount >= 1); -#if 0 - SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; +static void multiTableQueryImpl(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; pQuery->rec.size = 0; int64_t st = taosGetTimestampUs(); - if (pQuery->intervalTime > 0 || + + if (isIntervalQuery(pQuery) || (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { - assert(pQuery->checkBufferInLoop == 0); - vnodeMultiMeterQueryProcessor(pQInfo); + multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBufferInLoop == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)); @@ -5267,30 +5191,24 @@ void qSuperTableQuery(void *pReadMsg) { vnodeSTableSeqProcessor(pQInfo); } - /* record the total elapsed time */ + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); - pQuery->status = isQueryKilled(pQInfo) ? 1 : 0; - -// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, -// pQInfo->query.interpoType); +// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); if (pQuery->rec.size == 0) { -// pQInfo->over = 1; -// dTrace("QInfo:%p over, %d meters queried, %d points are returned", pQInfo, pSupporter->numOfMeters, -// pQInfo->size); + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + dTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, numOfTables, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); } sem_post(&pQInfo->dataReady); -// vnodeDecRefCount(pQInfo); -#endif } -static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryTableMsg, SSqlFuncExprMsg *pExprMsg) { +static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { int32_t j = 0; - while (j < pQueryTableMsg->numOfCols) { - if (pExprMsg->colInfo.colId == pQueryTableMsg->colList[j].colId) { + while (j < pQueryMsg->numOfCols) { + if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { break; } @@ -5300,59 +5218,56 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryTableMsg, SSqlFuncEx return j; } -bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryTableMsg, SSqlFuncExprMsg *pExprMsg) { - int32_t j = getColumnIndexInSource(pQueryTableMsg, pExprMsg); - return j < pQueryTableMsg->numOfCols; +bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { + int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg); + return j < pQueryMsg->numOfCols; } -static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryTableMsg) { - if (pQueryTableMsg->intervalTime < 0) { - dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryTableMsg, pQueryTableMsg->intervalTime); - return -1; - } - - if (pQueryTableMsg->numOfCols <= 0 || pQueryTableMsg->numOfCols > TSDB_MAX_COLUMNS) { - dError("qmsg:%p illegal value of numOfCols %d", pQueryTableMsg, pQueryTableMsg->numOfCols); +static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { + if (pQueryMsg->intervalTime < 0) { + dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime); return -1; } - if (pQueryTableMsg->numOfTables <= 0) { - dError("qmsg:%p illegal value of numOfTables %d", pQueryTableMsg, pQueryTableMsg->numOfTables); + if (pQueryMsg->numOfCols <= 0 || pQueryMsg->numOfCols > TSDB_MAX_COLUMNS) { + dError("qmsg:%p illegal value of numOfCols %d", pQueryMsg, pQueryMsg->numOfCols); return -1; } - if (pQueryTableMsg->numOfGroupCols < 0) { - dError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryTableMsg, pQueryTableMsg->numOfGroupCols); + if (pQueryMsg->numOfTables <= 0) { + dError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables); return -1; } - if (pQueryTableMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryTableMsg->numOfOutputCols <= 0) { - dError("qmsg:%p illegal value of output columns %d", pQueryTableMsg, pQueryTableMsg->numOfOutputCols); + if (pQueryMsg->numOfGroupCols < 0) { + dError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryMsg, pQueryMsg->numOfGroupCols); return -1; } - if (pQueryTableMsg->tagLength < 0) { - dError("qmsg:%p illegal value of tag length %d", pQueryTableMsg, pQueryTableMsg->tagLength); + if (pQueryMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutputCols <= 0) { + dError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutputCols); return -1; } return 0; } -static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArray** pTableIdList) { - assert(pQueryTableMsg->numOfTables > 0); +static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** pTableIdList) { + assert(pQueryMsg->numOfTables > 0); - *pTableIdList = taosArrayInit(pQueryTableMsg->numOfTables, sizeof(STableIdInfo)); + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableId)); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - taosArrayPush(*pTableIdList, pTableIdInfo); + STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->sid}; + taosArrayPush(*pTableIdList, &id); + pMsg += sizeof(STableIdInfo); - for (int32_t j = 1; j < pQueryTableMsg->numOfTables; ++j) { + for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) { pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->sid = htonl(pTableIdInfo->sid); @@ -5367,48 +5282,47 @@ static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArra } /** - * pQueryTableMsg->head has been converted before this function is called. + * pQueryMsg->head has been converted before this function is called. * - * @param pQueryTableMsg + * @param pQueryMsg * @param pTableIdList * @param pExpr * @return */ -static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr) { - pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); - - pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); - pQueryTableMsg->window.ekey = htobe64(pQueryTableMsg->window.ekey); - pQueryTableMsg->intervalTime = htobe64(pQueryTableMsg->intervalTime); - pQueryTableMsg->slidingTime = htobe64(pQueryTableMsg->slidingTime); +static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, + wchar_t** tagCond) { + pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); + + pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); + pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime); + pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); + pQueryMsg->limit = htobe64(pQueryMsg->limit); + pQueryMsg->offset = htobe64(pQueryMsg->offset); - pQueryTableMsg->limit = htobe64(pQueryTableMsg->limit); - pQueryTableMsg->offset = htobe64(pQueryTableMsg->offset); - - pQueryTableMsg->order = htons(pQueryTableMsg->order); - pQueryTableMsg->orderColId = htons(pQueryTableMsg->orderColId); - - pQueryTableMsg->queryType = htons(pQueryTableMsg->queryType); - - pQueryTableMsg->numOfCols = htons(pQueryTableMsg->numOfCols); - pQueryTableMsg->numOfOutputCols = htons(pQueryTableMsg->numOfOutputCols); - pQueryTableMsg->numOfGroupCols = htons(pQueryTableMsg->numOfGroupCols); - pQueryTableMsg->tagLength = htons(pQueryTableMsg->tagLength); - - pQueryTableMsg->tsOffset = htonl(pQueryTableMsg->tsOffset); - pQueryTableMsg->tsLen = htonl(pQueryTableMsg->tsLen); - pQueryTableMsg->tsNumOfBlocks = htonl(pQueryTableMsg->tsNumOfBlocks); - pQueryTableMsg->tsOrder = htonl(pQueryTableMsg->tsOrder); + pQueryMsg->order = htons(pQueryMsg->order); + pQueryMsg->orderColId = htons(pQueryMsg->orderColId); + pQueryMsg->queryType = htons(pQueryMsg->queryType); + + pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); + pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols); + pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); + pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->nameCondLen = htons(pQueryMsg->nameCondLen); + pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); + pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); + pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); + pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); // query msg safety check - if (validateQueryMeterMsg(pQueryTableMsg) != 0) { + if (validateQueryMsg(pQueryMsg) != 0) { return TSDB_CODE_INVALID_QUERY_MSG; } - char *pMsg = (char *)(pQueryTableMsg->colList) + sizeof(SColumnInfo) * pQueryTableMsg->numOfCols; + char *pMsg = (char *)(pQueryMsg->colList) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; - for (int32_t col = 0; col < pQueryTableMsg->numOfCols; ++col) { - SColumnInfo* pColInfo = &pQueryTableMsg->colList[col]; + for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { + SColumnInfo* pColInfo = &pQueryMsg->colList[col]; pColInfo->colId = htons(pColInfo->colId); pColInfo->type = htons(pColInfo->type); @@ -5448,11 +5362,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId bool hasArithmeticFunction = false; - *pExpr = calloc(pQueryTableMsg->numOfOutputCols, POINTER_BYTES); - + *pExpr = calloc(pQueryMsg->numOfOutputCols, POINTER_BYTES); SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; - for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { (*pExpr)[i] = pExprMsg; pExprMsg->colInfo.colIdx = htons(pExprMsg->colInfo.colIdx); @@ -5483,7 +5396,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId return TSDB_CODE_INVALID_QUERY_MSG; } } else { - if (!vnodeValidateExprColumnInfo(pQueryTableMsg, pExprMsg)) { + if (!vnodeValidateExprColumnInfo(pQueryMsg, pExprMsg)) { return TSDB_CODE_INVALID_QUERY_MSG; } } @@ -5491,47 +5404,55 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId pExprMsg = (SSqlFuncExprMsg *)pMsg; } - pQueryTableMsg->colNameLen = htonl(pQueryTableMsg->colNameLen); + pQueryMsg->colNameLen = htonl(pQueryMsg->colNameLen); if (hasArithmeticFunction) { // column name array - assert(pQueryTableMsg->colNameLen > 0); - pQueryTableMsg->colNameList = (int64_t)pMsg; - pMsg += pQueryTableMsg->colNameLen; + assert(pQueryMsg->colNameLen > 0); + pQueryMsg->colNameList = (int64_t)pMsg; + pMsg += pQueryMsg->colNameLen; } - pMsg = createTableIdList(pQueryTableMsg, pMsg, pTableIdList); + pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList); - if (pQueryTableMsg->numOfGroupCols > 0) { // group by tag columns -// if (pQueryTableMsg->numOfGroupCols > 0) { -// pQueryTableMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryTableMsg->numOfTagsCols]); + if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns +// if (pQueryMsg->numOfGroupCols > 0) { +// pQueryMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryMsg->numOfTagsCols]); // } else { -// pQueryTableMsg->groupbyTagIds = 0; +// pQueryMsg->groupbyTagIds = 0; // } - pQueryTableMsg->orderByIdx = htons(pQueryTableMsg->orderByIdx); - pQueryTableMsg->orderType = htons(pQueryTableMsg->orderType); + pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); + pQueryMsg->orderType = htons(pQueryMsg->orderType); - pMsg += sizeof(SColIndexEx) * pQueryTableMsg->numOfGroupCols; + pMsg += sizeof(SColIndexEx) * pQueryMsg->numOfGroupCols; } else { - pQueryTableMsg->groupbyTagIds = 0; + pQueryMsg->groupbyTagIds = 0; } - pQueryTableMsg->interpoType = htons(pQueryTableMsg->interpoType); - if (pQueryTableMsg->interpoType != TSDB_INTERPO_NONE) { - pQueryTableMsg->defaultVal = (uint64_t)(pMsg); + pQueryMsg->interpoType = htons(pQueryMsg->interpoType); + if (pQueryMsg->interpoType != TSDB_INTERPO_NONE) { + pQueryMsg->defaultVal = (uint64_t)(pMsg); int64_t *v = (int64_t *)pMsg; - for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { v[i] = htobe64(v[i]); } + + pMsg += sizeof(int64_t) * pQueryMsg->numOfOutputCols; + } + + // the tag query condition expression string is located at the end of query msg + if (pQueryMsg->tagCondLen > 0) { + *tagCond = calloc(1, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); + memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); } dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, " "timestamp order:%d, tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptslen:%d, limit:%" PRId64 ", offset:%" PRId64, - pQueryTableMsg, pQueryTableMsg->numOfTables, pQueryTableMsg->window.skey, pQueryTableMsg->window.ekey, - pQueryTableMsg->numOfGroupCols, pQueryTableMsg->order, pQueryTableMsg->orderType, - pQueryTableMsg->orderByIdx, pQueryTableMsg->numOfOutputCols, - pQueryTableMsg->numOfCols, pQueryTableMsg->intervalTime, pQueryTableMsg->interpoType, pQueryTableMsg->tsLen, - pQueryTableMsg->limit, pQueryTableMsg->offset); + pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, + pQueryMsg->numOfGroupCols, pQueryMsg->order, pQueryMsg->orderType, + pQueryMsg->orderByIdx, pQueryMsg->numOfOutputCols, + pQueryMsg->numOfCols, pQueryMsg->intervalTime, pQueryMsg->interpoType, pQueryMsg->tsLen, + pQueryMsg->limit, pQueryMsg->offset); return 0; } @@ -5911,7 +5832,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->pos = -1; - dTrace("QInfo %p is allocated", pQInfo); + pQuery->window.skey = pQueryMsg->window.skey; + pQuery->window.ekey = pQueryMsg->window.ekey; + pQuery->lastKey = pQuery->window.skey; + + if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { + dError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); + goto _clean_memory; + } + + vnodeParametersSafetyCheck(pQuery); + + dTrace("qmsg:%p create QInfo:%p, QInfo created", pQueryMsg, pQInfo); return pQInfo; _clean_memory: @@ -5935,7 +5867,7 @@ _clean_memory: return NULL; } -bool isQInfoValid(void *param) { +static bool isValidQInfo(void *param) { SQInfo *pQInfo = (SQInfo *)param; if (pQInfo == NULL) { return false; @@ -5949,11 +5881,51 @@ bool isQInfoValid(void *param) { return (sig == (uint64_t)pQInfo); } -void vnodeFreeQInfo(SQInfo *pQInfo) { - if (!isQInfoValid(pQInfo)) { - return; +static void freeQInfo(SQInfo *pQInfo); +static int32_t initializeQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { + int32_t code = TSDB_CODE_SUCCESS; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + STSBuf *pTSBuf = NULL; + if (pQueryMsg->tsLen > 0) { // open new file to save the result + char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; + pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); + + tsBufResetPos(pTSBuf); + tsBufNextPos(pTSBuf); + } + + // only the successful complete requries the sem_post/over = 1 operations. + if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || + (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { + dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, + pQuery->window.ekey, pQuery->order.order); + + sem_post(&pQInfo->dataReady); + setQueryStatus(pQuery, QUERY_COMPLETED); + return TSDB_CODE_SUCCESS; } + // filter the qualified + if ((code = doInitializeQInfo(pQInfo, pTSBuf, tsdb, isSTable)) != TSDB_CODE_SUCCESS) { + goto _error; + } + + // dTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, + // pQInfo->refCount); + return code; + +_error: + // table query ref will be decrease during error handling + freeQInfo(pQInfo); + return code; +} + +static void freeQInfo(SQInfo *pQInfo) { + if (!isValidQInfo(pQInfo)) { + return; + } + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; setQueryKilled(pQInfo); @@ -5961,7 +5933,7 @@ void vnodeFreeQInfo(SQInfo *pQInfo) { for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { tfree(pQuery->sdata[col]); } - + // for (int col = 0; col < pQuery->numOfCols; ++col) { // vnodeFreeColumnInfo(&pQuery->colList[col].data); // } @@ -5969,142 +5941,164 @@ void vnodeFreeQInfo(SQInfo *pQInfo) { // if (pQuery->colList[0].colIdx != PRIMARYKEY_TIMESTAMP_COL_INDEX) { // tfree(pQuery->tsData); // } - + sem_destroy(&(pQInfo->dataReady)); vnodeQueryFreeQInfoEx(pQInfo); - + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; if (pColFilter->numOfFilters > 0) { tfree(pColFilter->pFilters); } } - + tfree(pQuery->pFilterInfo); tfree(pQuery->colList); tfree(pQuery->sdata); - + if (pQuery->pSelectExpr != NULL) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SSqlBinaryExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; - + if (pBinExprInfo->numOfCols > 0) { tfree(pBinExprInfo->pReqColumns); tSQLBinaryExprDestroy(&pBinExprInfo->pBinExpr, NULL); } } - + tfree(pQuery->pSelectExpr); } - + if (pQuery->defaultVal != NULL) { tfree(pQuery->defaultVal); } - + tfree(pQuery->pGroupbyExpr); tfree(pQuery); - + dTrace("QInfo:%p QInfo is freed", pQInfo); - + // destroy signature, in order to avoid the query process pass the object safety check memset(pQInfo, 0, sizeof(SQInfo)); tfree(pQInfo); } -static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, - SArray *pTableIdList, void* tsdb, SQInfo **pQInfo) { - int32_t code = TSDB_CODE_SUCCESS; - - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pSqlExprs, pTableIdList); - if (pQInfo == NULL) { - code = TSDB_CODE_SERV_OUT_OF_MEMORY; - goto _error; - } - - SQuery *pQuery = (*pQInfo)->runtimeEnv.pQuery; - dTrace("qmsg:%p create QInfo:%p, QInfo created", pQueryMsg, pQInfo); - - pQuery->window.skey = pQueryMsg->window.skey; - pQuery->window.ekey = pQueryMsg->window.ekey; - pQuery->lastKey = pQuery->window.skey; - - if (sem_init(&(*pQInfo)->dataReady, 0, 0) != 0) { - dError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); - code = TSDB_CODE_APP_ERROR; - goto _error; - } - - vnodeParametersSafetyCheck(pQuery); - - STSBuf *pTSBuf = NULL; - if (pQueryMsg->tsLen > 0) { // open new file to save the result - char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; - pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); - - tsBufResetPos(pTSBuf); - tsBufNextPos(pTSBuf); +static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + /* + * get the file size and set the numOfRows to be the file size, since for tsComp query, + * the returned row size is equalled to 1 + * TODO handle the case that the file is too large to send back one time + */ + if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { + struct stat fstat; + if (stat(pQuery->sdata[0]->data, &fstat) == 0) { + *numOfRows = fstat.st_size; + return fstat.st_size; + } else { + dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); + return 0; + } + } else { + return pQuery->rowSize * (*numOfRows); } +} - if ((code = initQInfo(*pQInfo, pTSBuf, tsdb)) != TSDB_CODE_SUCCESS) { - goto _error; +static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { + // the remained number of retrieved rows, not the interpolated result + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + // load data from file to msg buffer + if (isTSCompQuery(pQuery)) { + int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); + + // make sure file exist + if (FD_VALID(fd)) { + size_t s = lseek(fd, 0, SEEK_END); + dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); + + lseek(fd, 0, SEEK_SET); + read(fd, data, s); + close(fd); + + unlink(pQuery->sdata[0]->data); + } else { + dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, + pQuery->sdata[0]->data, strerror(errno)); + } + } else { + doCopyQueryResultToMsg(pQInfo, pQuery->rec.size, data); } - - // if (pQInfo->over == 1) { - // vnodeAddRefCount(pQInfo); // for retrieve procedure - // return pQInfo; - // } - - // dTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, - // pQInfo->refCount); - return code; - -_error: - // table query ref will be decrease during error handling - vnodeFreeQInfo(*pQInfo); - return code; + + pQuery->rec.total += pQuery->rec.size; + dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + + return TSDB_CODE_SUCCESS; + + // todo if interpolation exists, the result may be dump to client by several rounds } -int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, void* param, SQInfo **pQInfo) { - assert(pQueryTableMsg != NULL); +int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) { + assert(pQueryMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; SArray *pTableIdList = NULL; SSqlFuncExprMsg** pExprMsg = NULL; - if ((code = convertQueryMsg(pQueryTableMsg, &pTableIdList, &pExprMsg)) != TSDB_CODE_SUCCESS) { + wchar_t* tagCond = NULL; + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond)) != TSDB_CODE_SUCCESS) { return code; } - if (pQueryTableMsg->numOfTables <= 0) { - dError("Invalid number of tables to query, numOfTables:%d", pQueryTableMsg->numOfTables); + if (pQueryMsg->numOfTables <= 0) { + dError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); code = TSDB_CODE_INVALID_QUERY_MSG; goto _query_over; } // todo check vnode status if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) { - dError("qmsg:%p, SQueryTableMsg wrong format", pQueryTableMsg); + dError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); code = TSDB_CODE_INVALID_QUERY_MSG; goto _query_over; } SSqlFunctionExpr *pExprs = NULL; - if ((code = createSqlFunctionExprFromMsg(pQueryTableMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { + if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { goto _query_over; } - SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryTableMsg, &code); - if ((pGroupbyExpr == NULL && pQueryTableMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { + SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, &code); + if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { goto _query_over; } - - if (QUERY_IS_STABLE_QUERY(pQueryTableMsg->queryType)) { - // pObj->qhandle = vnodeQueryOnMultiMeters(pMeterObjList, pGroupbyExpr, pExprs, pQueryTableMsg, &code); + + // super table query + SArray* res = NULL; + bool isSTableQuery = false; + if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { + isSTableQuery = true; + + STableId* id = taosArrayGet(pTableIdList, 0); + id->uid = -1; + + res = tsdbQueryTableList(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen); + if (taosArrayGetSize(res) == 0) { // no qualified table in stable query in this vnode + code = TSDB_CODE_SUCCESS; + goto _query_over; + } } else { - code = createQInfo(pQueryTableMsg, pGroupbyExpr, pExprs, pTableIdList, tsdb, pQInfo); -// (*pQInfo)->param = param; + res = pTableIdList; } - + + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, res); + if ((*pQInfo) == NULL) { + code = TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + code = initializeQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); + _query_over: if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pTableIdList); @@ -6112,24 +6106,47 @@ _query_over: // if failed to add ref for all meters in this query, abort current query // if (code != TSDB_CODE_SUCCESS) { - // vnodeDecQueryRefCount(pQueryTableMsg, pMeterObjList, incNumber); + // vnodeDecQueryRefCount(pQueryMsg, pMeterObjList, incNumber); // } // - // tfree(pQueryTableMsg->pSqlFuncExprs); + // tfree(pQueryMsg->pSqlFuncExprs); // tfree(pMeterObjList); // ret = vnodeSendQueryRspMsg(pObj, code, pObj->qhandle); // - // tfree(pQueryTableMsg->pSidExtInfo); - // for(int32_t i = 0; i < pQueryTableMsg->numOfCols; ++i) { - // vnodeFreeColumnInfo(&pQueryTableMsg->colList[i]); + // tfree(pQueryMsg->pSidExtInfo); + // for(int32_t i = 0; i < pQueryMsg->numOfCols; ++i) { + // vnodeFreeColumnInfo(&pQueryMsg->colList[i]); // } // // atomic_fetch_add_32(&vnodeSelectReqNum, 1); return TSDB_CODE_SUCCESS; } +void qTableQuery(SQInfo *pQInfo) { + if (pQInfo == NULL || pQInfo->signature != pQInfo) { + dTrace("%p freed abort query", pQInfo); + return; + } + + if (isQueryKilled(pQInfo)) { + dTrace("QInfo:%p it is already killed, abort", pQInfo); + return; + } + + dTrace("QInfo:%p query task is launched", pQInfo); + + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + if (numOfTables == 1) { + singleTableQueryImpl(pQInfo); + } else { + multiTableQueryImpl(pQInfo); + } + + // vnodeDecRefCount(pQInfo); +} + int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo) { - if (pQInfo == NULL || !isQInfoValid(pQInfo)) { + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } @@ -6150,62 +6167,6 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo) { return (pQInfo->code >= 0)? pQInfo->code:(-pQInfo->code); } -static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - - /* - * get the file size and set the numOfRows to be the file size, since for tsComp query, - * the returned row size is equalled to 1 - * TODO handle the case that the file is too large to send back one time - */ - if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { - struct stat fstat; - if (stat(pQuery->sdata[0]->data, &fstat) == 0) { - *numOfRows = fstat.st_size; - return fstat.st_size; - } else { - dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); - return 0; - } - } else { - return pQuery->rowSize * (*numOfRows); - } -} - -static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { - // the remained number of retrieved rows, not the interpolated result - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - // load data from file to msg buffer - if (isTSCompQuery(pQuery)) { - int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); - - // make sure file exist - if (FD_VALID(fd)) { - size_t s = lseek(fd, 0, SEEK_END); - dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); - - lseek(fd, 0, SEEK_SET); - read(fd, data, s); - close(fd); - - unlink(pQuery->sdata[0]->data); - } else { - dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, - pQuery->sdata[0]->data, strerror(errno)); - } - } else { - doCopyQueryResultToMsg(pQInfo, pQuery->rec.size, data); - } - - pQuery->rec.total += pQuery->rec.size; - dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); - - return TSDB_CODE_SUCCESS; - - // todo if interpolation exists, the result may be dump to client by several rounds -} - bool qHasMoreResultsToRetrieve(SQInfo* pQInfo) { if (pQInfo == NULL || pQInfo->signature != pQInfo || pQInfo->code != TSDB_CODE_SUCCESS) { return false; @@ -6224,7 +6185,7 @@ bool qHasMoreResultsToRetrieve(SQInfo* pQInfo) { } int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen) { - if (pQInfo == NULL || !isQInfoValid(pQInfo)) { + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } @@ -6253,7 +6214,7 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { (*pRsp)->completed = 1; // notify no more result to client - vnodeFreeQInfo(pQInfo); + freeQInfo(pQInfo); } return code; diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0ae86007568d3d7a00ba6f7b147ac5f0d00ee74a --- /dev/null +++ b/src/query/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) + +IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) + MESSAGE(STATUS "gTest library found, build unit test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) +ENDIF() \ No newline at end of file diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c23f0f59241d7bbc7527f7fe332fe97b666acbcc --- /dev/null +++ b/src/query/tests/histogramTest.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "tstoken.h" +#include "tutil.h" + +#include "qhistogram.h" + +/* test validate the names for table/database */ +TEST(testCase, histogram_binary_search) { + SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); + + pHisto->numOfEntries = 10; + for (int32_t i = 0; i < 10; ++i) { + pHisto->elems[i].num = 1; + pHisto->elems[i].val = i; + } + + int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1); + assert(idx == 1); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9); + assert(idx == 9); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20); + assert(idx == 10); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1); + assert(idx == 0); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); + assert(idx == 4); + + free(pHisto); +} + +TEST(testCase, histogram_add) { + SHistogramInfo* pHisto = NULL; + + /** + * use arrayList, elapsed time is: + * before: + * 10,000,000 45sec, bin:1000 (-O0) / 17sec. bin:1000, (-O3) + * + * after: + * + */ + struct timeval systemTime; + gettimeofday(&systemTime, NULL); + int64_t st = + (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; + for (int32_t i = 0; i < 10000; ++i) { + tHistogramAdd(&pHisto, i); + // tHistogramPrint(pHisto); + } + // + gettimeofday(&systemTime, NULL); + int64_t et = + (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; + printf("total elapsed time: %ld\n", et - st); + + printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries); + tHistogramPrint(pHisto); + + printf("%ld\n", tHistogramSum(pHisto, 1.5)); + printf("%ld\n", tHistogramSum(pHisto, 2)); + printf("%ld\n", tHistogramSum(pHisto, 3)); + printf("%ld\n", tHistogramSum(pHisto, 4)); + printf("%ld\n", tHistogramSum(pHisto, 5)); + printf("%ld\n", tHistogramSum(pHisto, 6)); + + for (int32_t i = 399; i < 400; ++i) { + printf("val:%d, %ld\n", i, tHistogramSum(pHisto, i)); + } + + double ratio[] = {0 / 100, 20.0 / 100, 88.0 / 100, 100 / 100}; + double* res = tHistogramUniform(pHisto, ratio, 4); + for (int32_t i = 0; i < 4; ++i) { + printf("%f\n", res[i]); + } + + SHistogramInfo* pHisto1 = NULL; + for (int32_t i = (90000 - 1); i >= 80000; --i) { + tHistogramAdd(&pHisto1, i); + } + tHistogramPrint(pHisto1); + + SHistogramInfo* pRes = tHistogramMerge(pHisto1, pHisto, MAX_HISTOGRAM_BIN); + assert(pRes->numOfElems == pHisto->numOfElems + pHisto1->numOfElems); + tHistogramPrint(pRes); + + tHistogramDestroy(&pHisto); + tHistogramDestroy(&pHisto1); + tHistogramDestroy(&pRes); + free(res); +} + +TEST(testCase, heapsort) { + // int32_t num = 20; + // + // SHeapEntry* pEntry = tHeapCreate(num); + // + // for(int32_t i=0; i +#include +#include +#include + +#include "tsqlfunction.h" + +TEST(testCase, patternMatchTest) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + const char* str = "abcdef"; + int32_t ret = patternMatch("a%b%", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tm01"; + ret = patternMatch("tm__", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tkm1"; + ret = patternMatch("t%m1", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tkm1"; + ret = patternMatch("%m1", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = ""; + ret = patternMatch("%_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = "1"; + ret = patternMatch("%__", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = ""; + ret = patternMatch("%", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = " "; + ret = patternMatch("_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "!"; + ret = patternMatch("%_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefg"; + ret = patternMatch("abc%fg", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%fg", str, 7, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%f_", str, 6, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%f_", str, 1, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("ab", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("a%", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("a__", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); +} diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fca9ac8f9d82911975dc9e917b48b93be26142de --- /dev/null +++ b/src/query/tests/resultBufferTest.cpp @@ -0,0 +1,35 @@ +#include +#include +#include + +#include "taos.h" +#include "qresultBuf.h" +#include "tsdb.h" + +namespace { +// simple test +void simpleTest() { + SDiskbasedResultBuf* pResultBuf = NULL; + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64); + + int32_t pageId = 0; + int32_t groupId = 0; + + tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + ASSERT_TRUE(pBufPage != NULL); + + ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64); + ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L); + + SIDList list = getDataBufPagesIdList(pResultBuf, groupId); + ASSERT_EQ(list.size, 1); + + ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); + + destroyResultBuf(pResultBuf); +} +} // namespace + +TEST(testCase, resultBufferTest) { + simpleTest(); +} diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2392bd00b53c52126585d5bf1e795fe8db15f106 --- /dev/null +++ b/src/query/tests/tsBufTest.cpp @@ -0,0 +1,451 @@ +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "tstoken.h" +#include "ttime.h" +#include "tutil.h" +#include "qtsbuf.h" + +namespace { +/** + * + * @param num total number + * @param step gap between two consecutive ts + * @return + */ +int64_t* createTsList(int32_t num, int64_t start, int32_t step) { + int64_t* pList = (int64_t*)malloc(num * sizeof(int64_t)); + + for (int64_t i = 0; i < num; ++i) { + pList[i] = start + i * step; + } + + return pList; +} + +// simple test +void simpleTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + // write 10 ts points + int32_t num = 10; + int64_t tag = 1; + + int64_t* list = createTsList(10, 10000000, 30); + tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); + EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +// one large list of ts, the ts list need to be split into several small blocks +void largeTSTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + // write 10 ts points + int32_t num = 1000000; + int64_t tag = 1; + + int64_t* list = createTsList(num, 10000000, 30); + tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + + // the data has been flush to disk, no data in cache + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void multiTagsTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t tag = 1; + int64_t start = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void multiVnodeTagsTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t start = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + + // 2000 vnodes + for (int32_t j = 0; j < 20; ++j) { + // vnodeId:0 + start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void loadDataTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t oldStart = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + int32_t numOfVnode = 200; + + // 10000 vnodes + for (int32_t j = 0; j < numOfVnode; ++j) { + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %lld\n", i, list[0]); + + free(list); + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + // create from exists file + STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false); + EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder); + EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode); + EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize); + + EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset); + EXPECT_EQ(pNewBuf->pData[0].info.numOfBlocks, pTSBuf->pData[0].info.numOfBlocks); + EXPECT_EQ(pNewBuf->pData[0].info.compLen, pTSBuf->pData[0].info.compLen); + + EXPECT_STREQ(pNewBuf->path, pTSBuf->path); + + tsBufResetPos(pNewBuf); + + int64_t s = taosGetTimestampUs(); + printf("start:%lld\n", s); + + int32_t x = 0; + while (tsBufNextPos(pNewBuf)) { + STSElem elem = tsBufGetElem(pNewBuf); + if (++x == 100000000) { + break; + } + + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + int64_t e = taosGetTimestampUs(); + printf("end:%lld, elapsed:%lld, total obj:%d\n", e, e - s, x); +} + +void randomIncTsTest() {} + +void TSTraverse() { + // 10000 vnodes + int32_t num = 200000; + int64_t oldStart = 10000000; + int32_t numOfTags = 3; + int32_t step = 30; + int32_t numOfVnode = 2; + + STSBuf* pTSBuf = tsBufCreate(true); + + for (int32_t j = 0; j < numOfVnode; ++j) { + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]); + + free(list); + start += step * num; + + list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + tsBufResetPos(pTSBuf); + + //////////////////////////////////////////////////////////////////////////////////////// + // reverse traverse + int64_t s = taosGetTimestampUs(); + printf("start:%lld\n", s); + + pTSBuf->cur.order = TSQL_SO_DESC; + + // complete reverse traverse + int32_t x = 0; + while (tsBufNextPos(pTSBuf)) { + STSElem elem = tsBufGetElem(pTSBuf); + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + // specify the data block with vnode and tags value + tsBufResetPos(pTSBuf); + pTSBuf->cur.order = TSQL_SO_DESC; + + int32_t startVnode = 1; + int32_t startTag = 2; + + tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + + int32_t totalOutput = 10; + while (1) { + STSElem elem = tsBufGetElem(pTSBuf); + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + + if (!tsBufNextPos(pTSBuf)) { + break; + } + + if (--totalOutput <= 0) { + totalOutput = 10; + + tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + + if (startTag == 0) { + startVnode -= 1; + startTag = 3; + } + + if (startVnode < 0) { + break; + } + } + } + + ///////////////////////////////////////////////////////////////////////////////// + // traverse + pTSBuf->cur.order = TSQL_SO_ASC; + tsBufResetPos(pTSBuf); + + // complete forwards traverse + while (tsBufNextPos(pTSBuf)) { + STSElem elem = tsBufGetElem(pTSBuf); + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + // specify the data block with vnode and tags value + tsBufResetPos(pTSBuf); + pTSBuf->cur.order = TSQL_SO_ASC; + + startVnode = 1; + startTag = 2; + + tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + + totalOutput = 10; + while (1) { + STSElem elem = tsBufGetElem(pTSBuf); + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + + if (!tsBufNextPos(pTSBuf)) { + break; + } + + if (--totalOutput <= 0) { + totalOutput = 10; + + tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + + if (startTag < 0) { + startVnode -= 1; + startTag = 3; + } + + if (startVnode < 0) { + break; + } + } + } +} + +void performanceTest() {} + +void emptyTagTest() {} + +void invalidFileTest() { + const char* cmd = "touch /tmp/test"; + + // create empty file + system(cmd); + + STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true); + EXPECT_TRUE(pNewBuf == NULL); + + pNewBuf = tsBufCreateFromFile("/tmp/911", true); + EXPECT_TRUE(pNewBuf == NULL); +} + +void mergeDiffVnodeBufferTest() { + STSBuf* pTSBuf1 = tsBufCreate(true); + STSBuf* pTSBuf2 = tsBufCreate(true); + + int32_t step = 30; + int32_t num = 1000; + int32_t numOfTags = 10; + + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t)); + + free(list); + + start += step * num; + } + + tsBufFlush(pTSBuf2); + + tsBufMerge(pTSBuf1, pTSBuf2, 9); + EXPECT_EQ(pTSBuf1->numOfVnodes, 2); + EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); + + tsBufDisplay(pTSBuf1); + + tsBufDestory(pTSBuf2); + tsBufDestory(pTSBuf1); +} + +void mergeIdenticalVnodeBufferTest() { + STSBuf* pTSBuf1 = tsBufCreate(true); + STSBuf* pTSBuf2 = tsBufCreate(true); + + int32_t step = 30; + int32_t num = 1000; + int32_t numOfTags = 10; + + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + + tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { + int64_t* list = createTsList(num, start, step); + + tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + tsBufFlush(pTSBuf2); + + tsBufMerge(pTSBuf1, pTSBuf2, 12); + EXPECT_EQ(pTSBuf1->numOfVnodes, 1); + EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); + + tsBufResetPos(pTSBuf1); + while (tsBufNextPos(pTSBuf1)) { + STSElem elem = tsBufGetElem(pTSBuf1); + EXPECT_EQ(elem.vnode, 12); + + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + tsBufDestory(pTSBuf1); + tsBufDestory(pTSBuf2); +} +} // namespace + +TEST(testCase, tsBufTest) { + simpleTest(); + largeTSTest(); + multiTagsTest(); + multiVnodeTagsTest(); + loadDataTest(); + invalidFileTest(); + // randomIncTsTest(); + TSTraverse(); + mergeDiffVnodeBufferTest(); + mergeIdenticalVnodeBufferTest(); +} diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c7520c90f9fbdb818d85d8f3b321c49820441f2d --- /dev/null +++ b/src/query/tests/unitTest.cpp @@ -0,0 +1,769 @@ +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "../../client/inc/tscUtil.h" +#include "ttime.h" +#include "tutil.h" +#include "tvariant.h" +#include "ttokendef.h" + +namespace { +int32_t testValidateName(char* name) { + SSQLToken token = {0}; + token.z = name; + token.n = strlen(name); + token.type = 0; + + tSQLGetToken(name, &token.type); + return tscValidateName(&token); +} +} + +static void _init_tvariant_bool(tVariant* t) { + t->i64Key = TSDB_FALSE; + t->nType = TSDB_DATA_TYPE_BOOL; +} + +static void _init_tvariant_tinyint(tVariant* t) { + t->i64Key = -27; + t->nType = TSDB_DATA_TYPE_TINYINT; +} + +static void _init_tvariant_int(tVariant* t) { + t->i64Key = -23997659; + t->nType = TSDB_DATA_TYPE_INT; +} + +static void _init_tvariant_bigint(tVariant* t) { + t->i64Key = -3333333333333; + t->nType = TSDB_DATA_TYPE_BIGINT; +} + +static void _init_tvariant_float(tVariant* t) { + t->dKey = -8991212199.8987878776; + t->nType = TSDB_DATA_TYPE_FLOAT; +} + +static void _init_tvariant_binary(tVariant* t) { + tVariantDestroy(t); + + t->pz = (char*)calloc(1, 20); //"2e3"); + t->nType = TSDB_DATA_TYPE_BINARY; + strcpy(t->pz, "2e5"); + t->nLen = strlen(t->pz); +} + +static void _init_tvariant_nchar(tVariant* t) { + tVariantDestroy(t); + + t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE); + t->nType = TSDB_DATA_TYPE_NCHAR; + wcscpy(t->wpz, L"-2000000.8765"); + t->nLen = wcslen(t->wpz); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +/* test validate the names for table/database */ +TEST(testCase, db_table_name) { + + char t01[] = "abc"; + EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS); + + char t02[] = "'abc'"; + EXPECT_EQ(testValidateName(t02), TSDB_CODE_SUCCESS); + + char t1[] = "abc.def"; + EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS); + printf("%s\n", t1); + + char t2[] = "'abc.def'"; + EXPECT_EQ(testValidateName(t2), TSDB_CODE_SUCCESS); + printf("%s\n", t2); + + char t3[] = "'abc'.def"; + EXPECT_EQ(testValidateName(t3), TSDB_CODE_SUCCESS); + printf("%s\n", t3); + + char t4[] = "'abc'.'def'"; + EXPECT_EQ(testValidateName(t4), TSDB_CODE_SUCCESS); + + char t5[] = "table.'def'"; + EXPECT_EQ(testValidateName(t5), TSDB_CODE_INVALID_SQL); + + char t6[] = "'table'.'def'"; + EXPECT_EQ(testValidateName(t6), TSDB_CODE_INVALID_SQL); + + char t7[] = "'_ab1234'.'def'"; + EXPECT_EQ(testValidateName(t7), TSDB_CODE_SUCCESS); + printf("%s\n", t7); + + char t8[] = "'_ab&^%1234'.'def'"; + EXPECT_EQ(testValidateName(t8), TSDB_CODE_INVALID_SQL); + + char t9[] = "'_123'.'gtest中文'"; + EXPECT_EQ(testValidateName(t9), TSDB_CODE_INVALID_SQL); + + char t10[] = "abc.'gtest中文'"; + EXPECT_EQ(testValidateName(t10), TSDB_CODE_INVALID_SQL); + + char t10_1[] = "abc.'中文gtest'"; + EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_INVALID_SQL); + + char t11[] = "'192.168.0.1'.abc"; + EXPECT_EQ(testValidateName(t11), TSDB_CODE_INVALID_SQL); + + char t12[] = "192.168.0.1.abc"; + EXPECT_EQ(testValidateName(t12), TSDB_CODE_INVALID_SQL); + + char t13[] = "abc."; + EXPECT_EQ(testValidateName(t13), TSDB_CODE_INVALID_SQL); + + char t14[] = ".abc"; + EXPECT_EQ(testValidateName(t14), TSDB_CODE_INVALID_SQL); + + char t15[] = ".'abc'"; + EXPECT_EQ(testValidateName(t15), TSDB_CODE_INVALID_SQL); + + char t16[] = ".abc'"; + EXPECT_EQ(testValidateName(t16), TSDB_CODE_INVALID_SQL); + + char t17[] = "123a.\"abc\""; + EXPECT_EQ(testValidateName(t17), TSDB_CODE_INVALID_SQL); + printf("%s\n", t17); + + char t18[] = "a.\"abc\""; + EXPECT_EQ(testValidateName(t18), TSDB_CODE_SUCCESS); + printf("%s\n", t18); + + char t19[] = "'_ab1234'.'def'.'ab123'"; + EXPECT_EQ(testValidateName(t19), TSDB_CODE_INVALID_SQL); + + char t20[] = "'_ab1234*&^'"; + EXPECT_EQ(testValidateName(t20), TSDB_CODE_INVALID_SQL); + + char t21[] = "'1234_abc'"; + EXPECT_EQ(testValidateName(t21), TSDB_CODE_INVALID_SQL); + + + // =======Containing capital letters================= + char t30[] = "ABC"; + EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS); + + char t31[] = "'ABC'"; + EXPECT_EQ(testValidateName(t31), TSDB_CODE_SUCCESS); + + char t32[] = "ABC.def"; + EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS); + + char t33[] = "'ABC.def"; + EXPECT_EQ(testValidateName(t33), TSDB_CODE_INVALID_SQL); + + char t33_0[] = "abc.DEF'"; + EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_INVALID_SQL); + + char t34[] = "'ABC.def'"; + //int32_t tmp0 = testValidateName(t34); + EXPECT_EQ(testValidateName(t34), TSDB_CODE_SUCCESS); + + char t35[] = "'ABC'.def"; + EXPECT_EQ(testValidateName(t35), TSDB_CODE_SUCCESS); + + char t36[] = "'ABC'.'DEF'"; + EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS); + + char t37[] = "abc.'DEF'"; + EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS); + + char t37_1[] = "abc.'_123DEF'"; + EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS); + + char t38[] = "'abc'.'DEF'"; + EXPECT_EQ(testValidateName(t38), TSDB_CODE_SUCCESS); + + // do not use key words + char t39[] = "table.'DEF'"; + EXPECT_EQ(testValidateName(t39), TSDB_CODE_INVALID_SQL); + + char t40[] = "'table'.'DEF'"; + EXPECT_EQ(testValidateName(t40), TSDB_CODE_INVALID_SQL); + + char t41[] = "'_abXYZ1234'.'deFF'"; + EXPECT_EQ(testValidateName(t41), TSDB_CODE_SUCCESS); + + char t42[] = "'_abDEF&^%1234'.'DIef'"; + EXPECT_EQ(testValidateName(t42), TSDB_CODE_INVALID_SQL); + + char t43[] = "'_123'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t43), TSDB_CODE_INVALID_SQL); + + char t44[] = "'aABC'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t44), TSDB_CODE_INVALID_SQL); + + char t45[] = "'ABC'."; + EXPECT_EQ(testValidateName(t45), TSDB_CODE_INVALID_SQL); + + char t46[] = ".'ABC'"; + EXPECT_EQ(testValidateName(t46), TSDB_CODE_INVALID_SQL); + + char t47[] = "a.\"aTWc\""; + EXPECT_EQ(testValidateName(t47), TSDB_CODE_SUCCESS); + + // ================has space ================= + char t60[] = " ABC "; + EXPECT_EQ(testValidateName(t60), TSDB_CODE_INVALID_SQL); + + char t60_1[] = " ABC "; + EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_INVALID_SQL); + + char t61[] = "' ABC '"; + EXPECT_EQ(testValidateName(t61), TSDB_CODE_SUCCESS); + + char t61_1[] = "' ABC '"; + EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_SUCCESS); + + char t62[] = " ABC . def "; + EXPECT_EQ(testValidateName(t62), TSDB_CODE_INVALID_SQL); + + char t63[] = "' ABC . def "; + EXPECT_EQ(testValidateName(t63), TSDB_CODE_INVALID_SQL); + + char t63_0[] = " abc . DEF ' "; + EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_INVALID_SQL); + + char t64[] = " ' ABC . def ' "; + //int32_t tmp1 = testValidateName(t64); + EXPECT_EQ(testValidateName(t64), TSDB_CODE_INVALID_SQL); + + char t65[] = " ' ABC '. def "; + EXPECT_EQ(testValidateName(t65), TSDB_CODE_INVALID_SQL); + + char t66[] = "' ABC '.' DEF '"; + EXPECT_EQ(testValidateName(t66), TSDB_CODE_SUCCESS); + + char t67[] = "abc . ' DEF '"; + EXPECT_EQ(testValidateName(t67), TSDB_CODE_INVALID_SQL); + + char t68[] = "' abc '.' DEF '"; + EXPECT_EQ(testValidateName(t68), TSDB_CODE_SUCCESS); + + // do not use key words + char t69[] = "table.'DEF'"; + EXPECT_EQ(testValidateName(t69), TSDB_CODE_INVALID_SQL); + + char t70[] = "'table'.'DEF'"; + EXPECT_EQ(testValidateName(t70), TSDB_CODE_INVALID_SQL); + + char t71[] = "'_abXYZ1234 '.' deFF '"; + EXPECT_EQ(testValidateName(t71), TSDB_CODE_SUCCESS); + + char t72[] = "'_abDEF&^%1234'.' DIef'"; + EXPECT_EQ(testValidateName(t72), TSDB_CODE_INVALID_SQL); + + char t73[] = "'_123'.' Gtest中文'"; + EXPECT_EQ(testValidateName(t73), TSDB_CODE_INVALID_SQL); + + char t74[] = "' aABC'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t74), TSDB_CODE_INVALID_SQL); + + char t75[] = "' ABC '."; + EXPECT_EQ(testValidateName(t75), TSDB_CODE_INVALID_SQL); + + char t76[] = ".' ABC'"; + EXPECT_EQ(testValidateName(t76), TSDB_CODE_INVALID_SQL); + + char t77[] = " a . \"aTWc\" "; + EXPECT_EQ(testValidateName(t77), TSDB_CODE_INVALID_SQL); + + char t78[] = " a.\"aTWc \""; + EXPECT_EQ(testValidateName(t78), TSDB_CODE_INVALID_SQL); + + + // ===============muti string by space =================== + // There's no such case. + //char t160[] = "A BC"; + //EXPECT_EQ(testValidateName(t160), TSDB_CODE_INVALID_SQL); + //printf("end:%s\n", t160); + + // There's no such case. + //char t161[] = "' A BC '"; + //EXPECT_EQ(testValidateName(t161), TSDB_CODE_INVALID_SQL); + + char t162[] = " AB C . de f "; + EXPECT_EQ(testValidateName(t162), TSDB_CODE_INVALID_SQL); + + char t163[] = "' AB C . de f "; + EXPECT_EQ(testValidateName(t163), TSDB_CODE_INVALID_SQL); + + char t163_0[] = " ab c . DE F ' "; + EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_INVALID_SQL); + + char t164[] = " ' AB C . de f ' "; + //int32_t tmp2 = testValidateName(t164); + EXPECT_EQ(testValidateName(t164), TSDB_CODE_INVALID_SQL); + + char t165[] = " ' A BC '. de f "; + EXPECT_EQ(testValidateName(t165), TSDB_CODE_INVALID_SQL); + + char t166[] = "' AB C '.' DE F '"; + EXPECT_EQ(testValidateName(t166), TSDB_CODE_INVALID_SQL); + + char t167[] = "ab c . ' D EF '"; + EXPECT_EQ(testValidateName(t167), TSDB_CODE_INVALID_SQL); + + char t168[] = "' a bc '.' DE F '"; + EXPECT_EQ(testValidateName(t168), TSDB_CODE_INVALID_SQL); + +} + +/* test parse time function */ +TEST(testCase, parse_time) { + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + char t1[] = "2018-1-1 1:1:1.952798"; + char t13[] = "1970-1-1 0:0:0"; + + int64_t time = 0, time1 = 0; + + taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); + + char t2[] = "2018-1-1T1:1:1.952Z"; + taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI); + + EXPECT_EQ(time, 1514739661952 + 28800000); + + char t3[] = "2018-1-1 1:01:01.952"; + taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + char t4[] = "2018-1-1 1:01:01.9"; + char t5[] = "2018-1-1 1:01:1.900"; + char t6[] = "2018-01-01 1:1:1.90"; + char t7[] = "2018-01-01 01:01:01.9"; + char t8[] = "2018-01-01 01:01:01.9007865"; + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t9[] = "2017-4-3 1:1:2.980"; + char t10[] = "2017-4-3T2:1:2.98+9:00"; + taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t11[] = "2017-4-3T2:1:2.98+09:00"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t12[] = "2017-4-3T2:1:2.98+0900"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taos_options(TSDB_OPTION_TIMEZONE, "UTC"); + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + char t14[] = "1970-1-1T0:0:0Z"; + taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + char t40[] = "1970-1-1 0:0:0.999999999"; + taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); + + char t41[] = "1997-1-1 0:0:0.999999999"; + taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999); + + int64_t k = timezone; + char t42[] = "1997-1-1T0:0:0.999999999Z"; + taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); + + //////////////////////////////////////////////////////////////////// + // illegal timestamp format + char t15[] = "2017-12-33 0:0:0"; + EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1); + + char t16[] = "2017-12-31 99:0:0"; + EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1); + + char t17[] = "2017-12-31T9:0:0"; + EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1); + + char t18[] = "2017-12-31T9:0:0.Z"; + EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1); + + char t19[] = "2017-12-31 9:0:0.-1"; + EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1); + + char t20[] = "2017-12-31 9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0); + EXPECT_EQ(time, 1514682000100); + + char t21[] = "2017-12-31T9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1); + + char t22[] = "2017-12-31 9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0); + + char t23[] = "2017-12-31T9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0); +} + +TEST(testCase, tvariant_convert) { + // 1. bool data to all other data types + tVariant t = {0}; + _init_tvariant_bool(&t); + + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "FALSE"); + tVariantDestroy(&t); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"FALSE"); + tVariantDestroy(&t); + + // 2. tinyint to other data types + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-27"); + tVariantDestroy(&t); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-27"); + tVariantDestroy(&t); + + // 3. int to other data + // types////////////////////////////////////////////////////////////////// + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + EXPECT_EQ(t.i64Key, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-23997659"); + tVariantDestroy(&t); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-23997659"); + tVariantDestroy(&t); + + // 4. bigint to other data + // type////////////////////////////////////////////////////////////////////////////// + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-3333333333333"); + tVariantDestroy(&t); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-3333333333333"); + tVariantDestroy(&t); + + // 5. float to other data + // types//////////////////////////////////////////////////////////////////////// + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -8991212199); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-8991212199.898788"); + tVariantDestroy(&t); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-8991212199.898788"); + tVariantDestroy(&t); + + // 6. binary to other data types + // ////////////////////////////////////////////////////////////////// + t.pz = "true"; + t.nLen = strlen(t.pz); + t.nType = TSDB_DATA_TYPE_BINARY; + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "2e5"); + tVariantDestroy(&t); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"2e5"); + tVariantDestroy(&t); + + // 7. nchar to other data types + // ////////////////////////////////////////////////////////////////// + t.wpz = L"FALSE"; + t.nLen = wcslen(t.wpz); + t.nType = TSDB_DATA_TYPE_NCHAR; + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_nchar(&t); + EXPECT_LE(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -2000000); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-2000000.8765"); + tVariantDestroy(&t); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-2000000.8765"); + tVariantDestroy(&t); +} + +TEST(testCase, tGetToken_Test) { + char* s = ".123 "; + uint32_t type = 0; + + int32_t len = tSQLGetToken(s, &type); + EXPECT_EQ(type, TK_FLOAT); + EXPECT_EQ(len, strlen(s) - 1); + + char s1[] = "1.123e10 "; + len = tSQLGetToken(s1, &type); + EXPECT_EQ(type, TK_FLOAT); + EXPECT_EQ(len, strlen(s1) - 1); + + char s4[] = "0xff "; + len = tSQLGetToken(s4, &type); + EXPECT_EQ(type, TK_HEX); + EXPECT_EQ(len, strlen(s4) - 1); + + // invalid data type + char s2[] = "e10 "; + len = tSQLGetToken(s2, &type); + EXPECT_FALSE(type == TK_FLOAT); + + char s3[] = "1.1.1.1"; + len = tSQLGetToken(s3, &type); + EXPECT_EQ(type, TK_IPTOKEN); + EXPECT_EQ(len, strlen(s3)); + + char s5[] = "0x "; + len = tSQLGetToken(s5, &type); + EXPECT_FALSE(type == TK_HEX); +} + +static SSQLToken createStrToken(char* s) { + SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; + t.type = TK_STRING; + t.z = s; + t.n = strlen(s); + + return t; +} + +TEST(testCase, isValidNumber_test) { + SSQLToken t1 = createStrToken("123abc"); + + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("0xabc"); + EXPECT_EQ(isValidNumber(&t1), TK_HEX); + + t1 = createStrToken("0b11101"); + EXPECT_EQ(isValidNumber(&t1), TK_BIN); + + t1 = createStrToken(".134abc"); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("1e1 "); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("1+2"); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("-0x123"); + EXPECT_EQ(isValidNumber(&t1), TK_HEX); + + t1 = createStrToken("-1"); + EXPECT_EQ(isValidNumber(&t1), TK_INTEGER); + + t1 = createStrToken("-0b1110"); + EXPECT_EQ(isValidNumber(&t1), TK_BIN); + + t1 = createStrToken("-.234"); + EXPECT_EQ(isValidNumber(&t1), TK_FLOAT); +} + +TEST(testCase, getTempFilePath_test) { + char path[4096] = {0}; + memset(path, 1, 4096); + + getTmpfilePath("new_tmp", path); + printf("%s\n", path); +} + diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 5b812bcb7f42ea61d21812137dafdd762d3145f6..6c8f94ec392d3c6b3afe201d661f9e6e99fe9078 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -24,6 +24,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) ELSE () MESSAGE(STATUS "Failed to find iconv, use default encoding method") ENDIF () + + ADD_SUBDIRECTORY(tests) ELSEIF (TD_WINDOWS_64) ADD_DEFINITIONS(-DUSE_LIBICONV) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) @@ -108,4 +110,3 @@ ENDIF() #ENDIF () - diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 55bdc849adf4bf6d8941f61c669f73cdcf2880fd..6b70780d10482610ef64b1c50640756db4204bd6 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -87,7 +87,21 @@ size_t taosArrayGetSize(const SArray* pArray); void* taosArrayInsert(SArray* pArray, size_t index, void* pData); /** - * + * remove data entry of the given index + * @param pArray + * @param index + */ +void taosArrayRemove(SArray* pArray, size_t index); + +/** + * copy the whole array from source to destination + * @param pDst + * @param pSrc + */ +void taosArrayCopy(SArray* pDst, SArray* pSrc); + +/** + * destroy array list * @param pArray */ void taosArrayDestroy(SArray* pArray); diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 0ec1d0eab52298913029f455e1566d9cf5d6630b..18404f89c27d2c122e813edae674bb3869c451b7 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -51,7 +51,7 @@ typedef struct SSkipListNode { #define SL_GET_BACKWARD_POINTER(n, _l) \ ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)] -#define SL_GET_NODE_DATA(n) ((char*)(n) + SL_NODE_HEADER_SIZE((n)->level)) +#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level)) #define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n))) #define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n)) @@ -106,25 +106,25 @@ typedef struct tSkipListState { typedef struct SSkipListKeyInfo { uint8_t dupKey : 2; // if allow duplicated key in the skip list - uint8_t type : 6; // key type + uint8_t type : 4; // key type + uint8_t freeNode:2; // free node when destroy the skiplist uint8_t len; // maximum key length, used in case of string key } SSkipListKeyInfo; typedef struct SSkipList { - __compar_fn_t comparFn; - __sl_key_fn_t keyFn; - uint32_t size; - uint8_t maxLevel; - uint8_t level; - SSkipListKeyInfo keyInfo; - + __compar_fn_t comparFn; + __sl_key_fn_t keyFn; + uint32_t size; + uint8_t maxLevel; + uint8_t level; + SSkipListKeyInfo keyInfo; pthread_rwlock_t *lock; - SSkipListNode * pHead; - + SSkipListNode * pHead; // point to the first element + SSkipListNode * pTail; // point to the last element + void * lastKey; // last key in the skiplist #if SKIP_LIST_RECORD_PERFORMANCE tSkipListState state; // skiplist state #endif - } SSkipList; /* @@ -147,7 +147,7 @@ typedef struct SSkipListIterator { * @return */ SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe, - __sl_key_fn_t fn); + uint8_t freeNode, __sl_key_fn_t fn); /** * @@ -182,21 +182,28 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode); * @param keyType * @return */ -SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType); +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType); /** * get the size of skip list * @param pSkipList * @return */ -size_t tSkipListGetSize(const SSkipList* pSkipList); +size_t tSkipListGetSize(const SSkipList *pSkipList); + +/** + * display skip list of the given level, for debug purpose only + * @param pSkipList + * @param nlevel + */ +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); /** * create skiplist iterator * @param pSkipList * @return */ -SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList); +SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); /** * forward the skip list iterator @@ -217,7 +224,7 @@ SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); * @param iter * @return */ -void* tSkipListDestroyIter(SSkipListIterator* iter); +void *tSkipListDestroyIter(SSkipListIterator *iter); /* * remove only one node of the pKey value. @@ -234,7 +241,6 @@ bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey); */ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); - #ifdef __cplusplus } #endif diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index df97dde5ac7b1440b2cdf2abab9f0243f84ddee5..b80aad1cebab3782ad7e2c4751275d854ef5f92c 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -176,6 +176,14 @@ uint32_t ip2uint(const char *const ip_addr); void taosSetAllocMode(int mode, const char* path, bool autoDump); void taosDumpMemoryLeak(); +#define TD_EQ 0x1 +#define TD_GT 0x2 +#define TD_LT 0x4 +#define TD_GE (TD_EQ | TD_GT) +#define TD_LE (TD_EQ | TD_LT) +void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, + int (*compar)(const void *, const void *), int flags); + #ifdef TAOS_MEM_CHECK void * taos_malloc(size_t size, const char *file, uint32_t line); diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index fb2dac827e13d4526b6b7a6c358a34dd575eb941..eb1d2133174b59a0ba93830a51986cbc78dc4c4b 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -128,6 +128,38 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { return dst; } +void taosArrayRemove(SArray* pArray, size_t index) { + assert(index < pArray->size); + + if (index == pArray->size - 1) { + taosArrayPop(pArray); + return; + } + + size_t remain = pArray->size - index - 1; + memmove(pArray->pData + index * pArray->elemSize, pArray->pData + (index + 1) * pArray->elemSize, remain * pArray->elemSize); + pArray->size -= 1; +} + +void taosArrayCopy(SArray* pDst, SArray* pSrc) { + assert(pSrc != NULL && pDst != NULL); + + if (pDst->capacity < pSrc->size) { + void* pData = realloc(pDst->pData, pSrc->size * pSrc->elemSize); + if (pData == NULL) { // todo handle oom + + } else { + pDst->pData = pData; + pDst->capacity = pSrc->size; + } + } + + memcpy(pDst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); + pDst->elemSize = pSrc->elemSize; + pDst->capacity = pSrc->size; + pDst->size = pSrc->size; +} + void taosArrayDestroy(SArray* pArray) { if (pArray == NULL) { return; diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index badcb7802f510b2978abace6b21a1098e1cdc44d..bdb12a59f9776eaf2924c6c08c35c1b204e2065a 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -100,6 +100,8 @@ SListNode *tdListPopHead(SList *list) { list->head = node->next; } list->numOfEles--; + node->next = NULL; + node->prev = NULL; return node; } @@ -113,6 +115,7 @@ SListNode *tdListPopTail(SList *list) { list->tail = node->prev; } list->numOfEles--; + node->next = node->prev = NULL; return node; } @@ -131,6 +134,7 @@ SListNode *tdListPopNode(SList *list, SListNode *node) { node->next->prev = node->prev; } list->numOfEles--; + node->next = node->prev = NULL; return node; } @@ -138,11 +142,10 @@ SListNode *tdListPopNode(SList *list, SListNode *node) { // Move all node elements from src to dst, the dst is assumed as an empty list void tdListMove(SList *src, SList *dst) { // assert(dst->eleSize == src->eleSize); - dst->numOfEles = src->numOfEles; - dst->head = src->head; - dst->tail = src->tail; - src->numOfEles = 0; - src->head = src->tail = NULL; + SListNode *node = NULL; + while ((node = tdListPopHead(src)) != NULL) { + tdListAppendNode(dst, node); + } } void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(target, node->data, list->eleSize); } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index c46f45fd37178147c7bb2903462290f861ac6189..11e1e740e04669f4f486c4e21e61ac174f57cdbd 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -51,28 +51,27 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { level = 1; pSkipList->level = 1; } else { - if (level > pSkipList->level && pSkipList->level < pSkipList->maxLevel) { - level = (++pSkipList->level); + if (level > pSkipList->level) { + if (pSkipList->level < pSkipList->maxLevel) { + level = (++pSkipList->level); + } else { + level = pSkipList->level; + } } } + + assert(level <= pSkipList->maxLevel); return level; } -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, int32_t level, SSkipListNode *pNode); - -void tSkipListDoRecordPut(SSkipList *pSkipList) { -#if SKIP_LIST_RECORD_PERFORMANCE - const int32_t MAX_RECORD_NUM = 1000; +#define DO_MEMSET_PTR_AREA(n) do {\ +int32_t _l = (n)->level;\ +memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\ +(n)->level = _l;\ +} while(0) - if (pSkipList->state.nInsertObjs == MAX_RECORD_NUM) { - pSkipList->state.nInsertObjs = 1; - pSkipList->state.nTotalStepsForInsert = 0; - pSkipList->state.nTotalElapsedTimeForInsert = 0; - } else { - pSkipList->state.nInsertObjs++; - } -#endif -} +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); +static SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode); int32_t compareInt32Val(const void *pLeft, const void *pRight) { int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); @@ -142,28 +141,12 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { } int32_t compareStrVal(const void *pLeft, const void *pRight) { - // SSkipListKey *pL = (SSkipListKey *)pLeft; - // SSkipListKey *pR = (SSkipListKey *)pRight; - // - // if (pL->nLen == 0 && pR->nLen == 0) { - // return 0; - // } - // - // // handle only one-side bound compare situation, there is only lower bound or only upper bound - // if (pL->nLen == -1) { - // return 1; // no lower bound, lower bound is minimum, always return -1; - // } else if (pR->nLen == -1) { - // return -1; // no upper bound, upper bound is maximum situation, always return 1; - // } - // - // int32_t ret = strcmp(((SSkipListKey *)pLeft)->pz, ((SSkipListKey *)pRight)->pz); - // - // if (ret == 0) { - // return 0; - // } else { - // return ret > 0 ? 1 : -1; - // } - return 0; + int32_t ret = strcmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } } int32_t compareWStrVal(const void *pLeft, const void *pRight) { @@ -280,8 +263,30 @@ static __compar_fn_t getKeyComparator(int32_t keyType) { return comparFn; } +static bool initForwardBackwardPtr(SSkipList* pSkipList) { + uint32_t maxLevel = pSkipList->maxLevel; + + // head info + pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel) * 2); + if (pSkipList->pHead == NULL) { + return false; + } + + pSkipList->pHead->level = pSkipList->maxLevel; + + // tail info + pSkipList->pTail = (SSkipListNode*) ((char*) pSkipList->pHead + SL_NODE_HEADER_SIZE(maxLevel)); + pSkipList->pTail->level = pSkipList->maxLevel; + + for(int32_t i = 0; i < maxLevel; ++i) { + SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; + SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; + } + + return true; +} SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t lock, - __sl_key_fn_t fn) { + uint8_t freeNode, __sl_key_fn_t fn) { SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); if (pSkipList == NULL) { return NULL; @@ -291,22 +296,24 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, ui maxLevel = MAX_SKIP_LIST_LEVEL; } - pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey}; - pSkipList->keyFn = fn; - + pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey, .freeNode = freeNode}; + pSkipList->keyFn = fn; pSkipList->comparFn = getKeyComparator(keyType); pSkipList->maxLevel = maxLevel; - pSkipList->level = 1; - - pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel)); - pSkipList->pHead->level = pSkipList->maxLevel; - + pSkipList->level = 1; + + if (!initForwardBackwardPtr(pSkipList)) { + tfree(pSkipList); + return NULL; + } + if (lock) { pSkipList->lock = calloc(1, sizeof(pthread_rwlock_t)); if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { tfree(pSkipList->pHead); tfree(pSkipList); + return NULL; } } @@ -348,16 +355,17 @@ void *tSkipListDestroy(SSkipList *pSkipList) { pthread_rwlock_wrlock(pSkipList->lock); } - SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); // pSkipList->pHead.pForward[0]; + SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); - while (pNode) { + while (pNode != pSkipList->pTail) { SSkipListNode *pTemp = pNode; pNode = SL_GET_FORWARD_POINTER(pNode, 0); - tfree(pTemp); + + if (pSkipList->keyInfo.freeNode) { + tfree(pTemp); + } } - tfree(pSkipList->pHead); - if (pSkipList->lock) { pthread_rwlock_unlock(pSkipList->lock); pthread_rwlock_destroy(pSkipList->lock); @@ -380,32 +388,34 @@ void tSkipListRandNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSi } SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { - if (pSkipList == NULL) { + if (pSkipList == NULL || pNode == NULL) { return NULL; } if (pSkipList->lock) { pthread_rwlock_wrlock(pSkipList->lock); } - - // record one node is put into skiplist - tSkipListDoRecordPut(pSkipList); - + + // the new key is greater than the last key of skiplist append it at last position + char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); + if (pSkipList->size == 0 || pSkipList->comparFn(pSkipList->lastKey, newDatakey) < 0) { + return tSkipListDoAppend(pSkipList, pNode); + } + + // find the appropriated position to insert data SSkipListNode *px = pSkipList->pHead; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; bool identical = false; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); - while (p != NULL) { + while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); - char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); // if the forward element is less than the specified key, forward one step int32_t ret = pSkipList->comparFn(key, newDatakey); if (ret < 0) { px = p; - p = SL_GET_FORWARD_POINTER(px, i); } else { if (identical == false) { @@ -415,10 +425,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { break; } } - -#if SKIP_LIST_RECORD_PERFORMANCE - pSkipList->state.nTotalStepsForInsert++; -#endif + forward[i] = px; } @@ -430,45 +437,59 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { return forward[0]; } + + tSkipListDoInsert(pSkipList, forward, pNode); + return pNode; +} -#if SKIP_LIST_RECORD_PERFORMANCE - recordNodeEachLevel(pSkipList, level); -#endif +void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { + DO_MEMSET_PTR_AREA(pNode); + + for (int32_t i = 0; i < pNode->level; ++i) { + SSkipListNode *x = forward[i]; + +// if (x != pSkipList->pTail) { + SL_GET_BACKWARD_POINTER(pNode, i) = x; - int32_t level = SL_GET_NODE_LEVEL(pNode); - tSkipListDoInsert(pSkipList, forward, level, pNode); + SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); +// if (next) { + SL_GET_BACKWARD_POINTER(next, i) = pNode; +// } + SL_GET_FORWARD_POINTER(pNode, i) = next; + SL_GET_FORWARD_POINTER(x, i) = pNode; +// } else { +// SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; +// } + } + atomic_add_fetch_32(&pSkipList->size, 1); - -#if SKIP_LIST_RECORD_PERFORMANCE - pSkipList->state.nTotalMemSize += getOneNodeSize(pKey, level); -#endif - if (pSkipList->lock) { pthread_rwlock_unlock(pSkipList->lock); } - - return pNode; } -void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, int32_t level, SSkipListNode *pNode) { - for (int32_t i = 0; i < level; ++i) { - SSkipListNode *x = forward[i]; - if (x != NULL) { - SL_GET_BACKWARD_POINTER(pNode, i) = x; - - SSkipListNode *pForward = SL_GET_FORWARD_POINTER(x, i); - if (pForward) { - SL_GET_BACKWARD_POINTER(pForward, i) = pNode; - } - - SL_GET_FORWARD_POINTER(pNode, i) = SL_GET_FORWARD_POINTER(x, i); - SL_GET_FORWARD_POINTER(x, i) = pNode; - } else { - SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; - SL_GET_BACKWARD_POINTER(pSkipList->pHead, i) = (pSkipList->pHead); - } +SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode) { + // do clear pointer area + DO_MEMSET_PTR_AREA(pNode); + + for(int32_t i = 0; i < pNode->level; ++i) { + SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i); + SL_GET_FORWARD_POINTER(prev, i) = pNode; + SL_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; + + SL_GET_BACKWARD_POINTER(pNode, i) = prev; + SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; } + + pSkipList->lastKey = SL_GET_NODE_KEY(pSkipList, pNode); + + atomic_add_fetch_32(&pSkipList->size, 1); + if (pSkipList->lock) { + pthread_rwlock_unlock(pSkipList->lock); + } + + return pNode; } SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) { @@ -691,89 +712,6 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // return NULL; //} // -// int32_t tSkipListIterateList(SSkipList *pSkipList, SSkipListNode ***pRes, bool (*fp)(SSkipListNode *, void *), -// void *param) { -// (*pRes) = (SSkipListNode **)calloc(1, POINTER_BYTES * pSkipList->nSize); -// if (NULL == *pRes) { -// pError("error skiplist %p, malloc failed", pSkipList); -// return -1; -// } -// -// pthread_rwlock_rdlock(&pSkipList->lock); -// SSkipListNode *pStartNode = pSkipList->pHead.pForward[0]; -// int32_t num = 0; -// -// for (int32_t i = 0; i < pSkipList->nSize; ++i) { -// if (pStartNode == NULL) { -// pError("error skiplist %p, required length:%d, actual length:%d", pSkipList, pSkipList->nSize, i - 1); -//#ifdef _DEBUG_VIEW -// tSkipListPrint(pSkipList, 1); -//#endif -// break; -// } -// -// if (fp == NULL || (fp != NULL && fp(pStartNode, param) == true)) { -// (*pRes)[num++] = pStartNode; -// } -// -// pStartNode = pStartNode->pForward[0]; -// } -// -// pthread_rwlock_unlock(&pSkipList->lock); -// -// if (num == 0) { -// free(*pRes); -// *pRes = NULL; -// } else if (num < pSkipList->nSize) { // free unused memory -// char *tmp = realloc((*pRes), num * POINTER_BYTES); -// assert(tmp != NULL); -// -// *pRes = (SSkipListNode **)tmp; -// } -// -// return num; -//} -// -// int32_t tSkipListIteratorReset(SSkipList *pSkipList, SSkipListIterator *iter) { -// if (pSkipList == NULL) { -// return -1; -// } -// -// iter->pSkipList = pSkipList; -// if (pSkipList->lock) { -// pthread_rwlock_rdlock(&pSkipList->lock); -// } -// iter->cur = NULL; // pSkipList->pHead.pForward[0]; -// iter->num = pSkipList->size; -// -// if (pSkipList->lock) { -// pthread_rwlock_unlock(&pSkipList->lock); -// } -// -// return 0; -//} -// -// bool tSkipListIteratorNext(SSkipListIterator *iter) { -// if (iter->num == 0 || iter->pSkipList == NULL) { -// return false; -// } -// -// SSkipList *pSkipList = iter->pSkipList; -// -// pthread_rwlock_rdlock(&pSkipList->lock); -// if (iter->cur == NULL) { -// iter->cur = pSkipList->pHead.pForward[0]; -// } else { -// iter->cur = iter->cur->pForward[0]; -// } -// -// pthread_rwlock_unlock(&pSkipList->lock); -// -// return iter->cur != NULL; -//} -// -// SSkipListNode *tSkipListIteratorGet(SSkipListIterator *iter) { return iter->cur; } -// // int32_t tSkipListRangeQuery(SSkipList *pSkipList, tSKipListQueryCond *pCond, SSkipListNode ***pRes) { // pSkipList->state.queryCount++; // SSkipListNode *pStart = tSkipListParQuery(pSkipList, &pCond->lowerBnd, pCond->lowerBndRelOptr); @@ -841,12 +779,20 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { } SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); - int32_t id = 1; - - while (p) { + + int32_t id = 1; + char* prev = NULL; + + while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); + if (prev != NULL) { + assert(pSkipList->comparFn(prev, key) < 0); + } + switch (pSkipList->keyInfo.type) { case TSDB_DATA_TYPE_INT: + fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key); + break; case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_BIGINT: @@ -862,7 +808,8 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { fprintf(stdout, "\n"); } + prev = SL_GET_NODE_KEY(pSkipList, p); + p = SL_GET_FORWARD_POINTER(p, nlevel - 1); - // p = p->pForward[nlevel - 1]; } } diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 9c384b25bace047cdd80fe2903c330dbe3ed85d7..cbd08954cc21ffd1f8a0e3cfc9076e0330860c42 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -617,3 +617,72 @@ char *taosCharsetReplace(char *charsetstr) { return strdup(charsetstr); } + +#define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx)) +void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, int (*compar)(const void *, const void *), int flags) { + // TODO: need to check the correctness of this function + int l = 0; + int r = nmemb; + int idx = 0; + int comparison; + + if (flags == TD_EQ) { + return bsearch(key, base, nmemb, size, compar); + } else if (flags == TD_GE) { + if ((*compar)(key, elePtrAt(base, size, 0)) <= 0) return elePtrAt(base, size, 0); + if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) > 0) return NULL; + + while (l < r) { + idx = (l + r) / 2; + comparison = (*compar)(key, elePtrAt(base, size, idx)); + if (comparison < 0) { + r = idx; + } else if (comparison > 0) { + l = idx + 1; + } else { + return elePtrAt(base, size, idx); + } + } + + if ((*compar)(key, elePtrAt(base, size, idx) < 0)) { + return elePtrAt(base, size, idx); + } else { + if (idx + 1 > nmemb - 1) { + return NULL; + } else { + return elePtrAt(base, size, idx + 1); + } + } + } else if (flags == TD_LE) { + if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) >= 0) return elePtrAt(base, size, nmemb - 1); + if ((*compar)(key, elePtrAt(base, size, 0)) < 0) return NULL; + + while (l < r) { + idx = (l + r) / 2; + comparison = (*compar)(key, elePtrAt(base, size, idx)); + if (comparison < 0) { + r = idx; + } else if (comparison > 0) { + l = idx + 1; + } else { + return elePtrAt(base, size, idx); + } + } + + if ((*compar)(key, elePtrAt(base, size, idx)) > 0) { + return elePtrAt(base, size, idx); + } else { + if (idx == 0) { + return NULL; + } else { + return elePtrAt(base, size, idx - 1); + } + } + + } else { + assert(0); + return NULL; + } + + return NULL; +} diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..042c925165985a285670f5df845708f92146b353 --- /dev/null +++ b/src/util/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) + +IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) + MESSAGE(STATUS "gTest library found, build unit test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(utilTest tutil gtest pthread) +ENDIF() \ No newline at end of file diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1902fef4b64d0b796a85d9f58e1f422fe0df59e8 --- /dev/null +++ b/src/util/tests/cacheTest.cpp @@ -0,0 +1,142 @@ +#include +#include +#include + +#include "taos.h" +//#include "tsdb.h" + +//#include "testCommon.h" +#include "tstoken.h" +#include "tutil.h" +#include "tcache.h" +#include "ttimer.h" +#include "ttime.h" + +namespace { +int32_t tsMaxMgmtConnections = 10000; +int32_t tsMaxMeterConnections = 200; +} +// test cache +TEST(testCase, client_cache_test) { + const int32_t REFRESH_TIME_IN_SEC = 2; + void* tscTmr = taosTmrInit (tsMaxMgmtConnections*2, 200, 6000, "TSC"); + SCacheObj* tscCacheHandle = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC); + + char* key1 = "test1"; + char* data1 = "test11"; + + char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1), 1); + sleep(REFRESH_TIME_IN_SEC+1); + + printf("obj is still valid: %s\n", cachedObj); + + char* data2 = "test22"; + taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + + /* the object is cleared by cache clean operation */ + cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2), 20); + printf("after updated: %s\n", cachedObj); + + printf("start to remove data from cache\n"); + taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + printf("end of removing data from cache\n"); + + getchar(); + + char* key3 = "test2"; + char* data3 = "kkkkkkk"; + + char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3), 1); + printf("%s\n", cachedObj2); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + + sleep(3); + char* d = (char*) taosCacheAcquireByName(tscCacheHandle, key3); +// assert(d == NULL); + + char* key5 = "test5"; + char* data5 = "data5kkkkk"; + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5), 20); + + char* data6= "new Data after updated"; + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6), 20); + printf("%s\n", cachedObj2); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); + + char* data7 = "add call update procedure"; + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7), 20); + printf("%s\n=======================================\n\n", cachedObj2); + + char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); + taosCacheRelease(tscCacheHandle, (void**) &cc, false); + + char* data8 = "ttft"; + char* key6 = "key6"; + + char* ft = (char*) taosCachePut(tscCacheHandle, key6, data8, strlen(data8), 20); + taosCacheRelease(tscCacheHandle, (void**) &ft, false); + + /** + * 140ns + */ + uint64_t startTime = taosGetTimestampUs(); + printf("Cache Performance Test\nstart time:%lld\n", startTime); + for(int32_t i=0; i<1000; ++i) { + char* dd = (char*) taosCacheAcquireByName(tscCacheHandle, key6); + if (dd != NULL) { +// printf("get the data\n"); + } else { + printf("data has been released\n"); + } + + taosCacheRelease(tscCacheHandle, (void**) &dd, false); + } + + uint64_t endTime = taosGetTimestampUs(); + int64_t el = endTime - startTime; + + printf("End of Test, %lld\nTotal Elapsed Time:%lld us.avg:%f us\n", endTime, el, el/1000.0); + + taosCacheCleanup(tscCacheHandle); +} + +TEST(testCase, cache_resize_test) { + const int32_t REFRESH_TIME_IN_SEC = 2; + void* tscTmr = taosTmrInit (1000*2, 200, 6000, "TSC"); + + auto* pCache = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC); + + char key[256] = {0}; + char data[1024] = "abcdefghijk"; + int32_t len = strlen(data); + + uint64_t startTime = taosGetTimestampUs(); + int32_t num = 10000; + + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "abc_%7d", i); + taosCachePut(pCache, key, data, len, 3600); + } + uint64_t endTime = taosGetTimestampUs(); + + printf("add %d object cost:%lld us, avg:%f us\n", num, endTime - startTime, (endTime-startTime)/(double)num); + + startTime = taosGetTimestampUs(); + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "abc_%7d", i); + void* k = taosCacheAcquireByName(pCache, key); + assert(k != 0); + } + endTime = taosGetTimestampUs(); + printf("retrieve %d object cost:%lld us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num); + + taosCacheCleanup(pCache); + taosMsleep(20000); + getchar(); +} \ No newline at end of file diff --git a/src/util/tests/hashTest.cpp b/src/util/tests/hashTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b3baedb69692ab4322d6a033dc1c529f3034f905 --- /dev/null +++ b/src/util/tests/hashTest.cpp @@ -0,0 +1,156 @@ +#include +#include +#include +#include + +#include "hash.h" +#include "taos.h" +#include "ttime.h" + +namespace { +// the simple test code for basic operations +void simpleTest() { + auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + // put 400 elements in the hash table + for(int32_t i = -200; i < 200; ++i) { + taosHashPut(hashTable, (const char*) &i, sizeof(int32_t), (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 400); + + for(int32_t i = 0; i < 200; ++i) { + char* p = (char*) taosHashGet(hashTable, (const char*) &i, sizeof(int32_t)); + ASSERT_TRUE(p != nullptr); + ASSERT_EQ(*reinterpret_cast(p), i); + } + + for(int32_t i = 1000; i < 2000; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 400); + + for(int32_t i = 0; i < 100; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 300); + + for(int32_t i = 100; i < 150; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 250); + taosHashCleanup(hashTable); +} + +void stringKeyTest() { + auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + char key[128] = {0}; + + // put 200 elements in the hash table + for(int32_t i = 0; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 1000); + + for(int32_t i = 0; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + char* p = (char*) taosHashGet(hashTable, key, len); + ASSERT_TRUE(p != nullptr); + + ASSERT_EQ(*reinterpret_cast(p), i); + } + + for(int32_t i = 500; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + taosHashRemove(hashTable, key, len); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 500); + + for(int32_t i = 0; i < 499; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + taosHashRemove(hashTable, key, len); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 1); + + taosHashCleanup(hashTable); +} + +void functionTest() { + +} + +/** + * evaluate the performance issue, by add 10million elements in to hash table in + * a single threads situation + */ +void noLockPerformanceTest() { + auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + char key[128] = {0}; + int32_t num = 5000000; + + int64_t st = taosGetTimestampUs(); + + // put 10M elements in the hash table + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), num); + + int64_t et = taosGetTimestampUs(); + printf("Elpased time:%" PRId64 " us to add %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num); + + st = taosGetTimestampUs(); + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + char* p = (char*) taosHashGet(hashTable, key, len); + ASSERT_TRUE(p != nullptr); + + ASSERT_EQ(*reinterpret_cast(p), i); + } + + et = taosGetTimestampUs(); + printf("Elpased time:%" PRId64 " us to fetch all %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num); + + printf("The maximum length of overflow linklist in hash table is:%d\n", taosHashGetMaxOverflowLinkLength(hashTable)); + taosHashCleanup(hashTable); +} + +void multithreadsTest() { + //todo +} + +// check the function robustness +void invalidOperationTest() { + +} + +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, hashTest) { +// simpleTest(); +// stringKeyTest(); +// noLockPerformanceTest(); +// multithreadsTest(); +} \ No newline at end of file diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c009231874de94530ffa18d15f745531c4daa94c --- /dev/null +++ b/src/util/tests/skiplistTest.cpp @@ -0,0 +1,355 @@ +#include +#include +#include +#include + +#include "taosmsg.h" +#include "tskiplist.h" +#include "ttime.h" +#include "tutil.h" + +namespace { + +char* getkey(const void* data) { return (char*)(data); } + +void doubleSkipListTest() { + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, true, getkey); + + double doubleVal[1000] = {0}; + int32_t size = 20000; + + printf("generated %d keys is: \n", size); + + for (int32_t i = 0; i < size; ++i) { + if (i < 1000) { + doubleVal[i] = i * 0.997; + } + + int32_t level = 0; + int32_t size = 0; + + tSkipListRandNodeInfo(pSkipList, &level, &size); + auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2); + d->level = level; + + double* key = (double*)SL_GET_NODE_KEY(pSkipList, d); + key[0] = i * 0.997; + key[1] = i * 0.997; + + tSkipListPut(pSkipList, d); + } + + printf("the first level of skip list is:\n"); + tSkipListPrint(pSkipList, 1); + +#if 0 + SSkipListNode **pNodes = NULL; + SSkipListKey sk; + for (int32_t i = 0; i < 100; ++i) { + sk.nType = TSDB_DATA_TYPE_DOUBLE; + int32_t idx = abs((i * rand()) % 1000); + + sk.dKey = doubleVal[idx]; + + int32_t size = tSkipListGets(pSkipList, &sk, &pNodes); + + printf("the query result size is: %d\n", size); + for (int32_t j = 0; j < size; ++j) { + printf("the result is: %lf\n", pNodes[j]->key.dKey); + } + + if (size > 0) { + tfree(pNodes); + } + } + +#endif + + printf("double test end...\n"); + tSkipListDestroy(pSkipList); +} + +void randKeyTest() { + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_INT, sizeof(int32_t), 0, false, true, getkey); + + int32_t size = 200000; + srand(time(NULL)); + + printf("generated %d keys is: \n", size); + + for (int32_t i = 0; i < size; ++i) { + int32_t level = 0; + int32_t s = 0; + + tSkipListRandNodeInfo(pSkipList, &level, &s); + auto d = (SSkipListNode*)calloc(1, s + sizeof(int32_t) * 2); + d->level = level; + + int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); + key[0] = rand() % 1000000000; + + key[1] = key[0]; + + tSkipListPut(pSkipList, d); + } + + printf("the first level of skip list is:\n"); + tSkipListPrint(pSkipList, 1); + + printf("the sec level of skip list is:\n"); + tSkipListPrint(pSkipList, 2); + + printf("the 5 level of skip list is:\n"); + tSkipListPrint(pSkipList, 5); + + tSkipListDestroy(pSkipList); +} +void stringKeySkiplistTest() { + const int32_t max_key_size = 12; + + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_BINARY, max_key_size, 0, false, true, getkey); + + int32_t level = 0; + int32_t headsize = 0; + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + auto pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + pNode->level = level; + + char* d = SL_GET_NODE_DATA(pNode); + strncpy(d, "nyse", 5); + + *(double*)(d + max_key_size) = 12; + + tSkipListPut(pSkipList, pNode); + + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + pNode->level = level; + + d = SL_GET_NODE_DATA(pNode); + strncpy(d, "beijing", 8); + + *(double*)(d + max_key_size) = 911; + + tSkipListPut(pSkipList, pNode); + + printf("level one------------------\n"); + tSkipListPrint(pSkipList, 1); + +#if 0 + SSkipListNode **pRes = NULL; + int32_t ret = tSkipListGets(pSkipList, &key1, &pRes); + + assert(ret == 1); + assert(strcmp(pRes[0]->key.pz, "beijing") == 0); + assert(pRes[0]->key.nType == TSDB_DATA_TYPE_BINARY); + + tSkipListDestroyKey(&key1); + tSkipListDestroyKey(&key); + + tSkipListDestroy(pSkipList); + + free(pRes); +#endif + + tSkipListDestroy(pSkipList); + + int64_t s = taosGetTimestampUs(); + pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_BINARY, 20, 0, false, true, getkey); + char k[256] = {0}; + + int32_t total = 10000; + for (int32_t i = 0; i < total; ++i) { + int32_t n = sprintf(k, "abc_%d_%d", i, i); + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + auto pNode = (SSkipListNode*)calloc(1, headsize + 20 + sizeof(double)); + pNode->level = level; + + char* d = SL_GET_NODE_DATA(pNode); + strncpy(d, k, strlen(k)); + + tSkipListPut(pSkipList, pNode); + } + + int64_t e = taosGetTimestampUs(); + printf("elapsed time:%lld us to insert %d data, avg:%f us\n", (e - s), total, (double)(e - s) / total); + + printf("level two------------------\n"); + tSkipListPrint(pSkipList, 1); + +#if 0 + SSkipListNode **pres = NULL; + + s = taosGetTimestampMs(); + for (int32_t j = 0; j < total; ++j) { + int32_t n = sprintf(k, "abc_%d_%d", j, j); + key = tSkipListCreateKey(TSDB_DATA_TYPE_BINARY, k, n); + + int32_t num = tSkipListGets(pSkipList, &key, &pres); + assert(num > 0); + + // tSkipListRemove(pSkipList, &key); + tSkipListRemoveNode(pSkipList, pres[0]); + + if (num > 0) { + tfree(pres); + } + } + + e = taosGetTimestampMs(); + printf("elapsed time:%lldms\n", e - s); +#endif + tSkipListDestroy(pSkipList); +} + +void skiplistPerformanceTest() { + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, false, getkey); + + int32_t size = 1000000; + int64_t prev = taosGetTimestampMs(); + int64_t s = prev; + + int32_t level = 0; + int32_t headsize = 0; + + int32_t unit = MAX_SKIP_LIST_LEVEL * POINTER_BYTES * 2 + sizeof(double) * 2 + sizeof(int16_t); + + char* total = (char*)calloc(1, unit * size); + char* p = total; + + for (int32_t i = 0; i < size; ++i) { + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + SSkipListNode* d = (SSkipListNode*)p; + p += headsize + sizeof(double) * 2; + + d->level = level; + double* v = (double*)SL_GET_NODE_DATA(d); + v[0] = i * 0.997; + v[1] = i * 0.997; + + tSkipListPut(pSkipList, d); + + if (i % 100000 == 0) { + int64_t cur = taosGetTimestampMs(); + + int64_t elapsed = cur - prev; + printf("add %d, elapsed time: %lld ms, avg elapsed:%f ms, total:%d\n", 100000, elapsed, elapsed / 100000.0, i); + prev = cur; + } + } + + int64_t e = taosGetTimestampMs(); + printf("total:%lld ms, avg:%f\n", e - s, (e - s) / (double)size); + printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level); + + assert(tSkipListGetSize(pSkipList) == size); + + // printf("the level of skiplist is:\n"); + // + // printf("level two------------------\n"); + // tSkipListPrint(pSkipList, 2); + // + // printf("level three------------------\n"); + // tSkipListPrint(pSkipList, 3); + // + // printf("level four------------------\n"); + // tSkipListPrint(pSkipList, 4); + // + // printf("level nine------------------\n"); + // tSkipListPrint(pSkipList, 10); + + int64_t st = taosGetTimestampMs(); +#if 0 + for (int32_t i = 0; i < 100000; i += 1) { + key.dKey = i * 0.997; + tSkipListRemove(pSkipList, &key); + } +#endif + + int64_t et = taosGetTimestampMs(); + printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st); + assert(tSkipListGetSize(pSkipList) == size); + + tSkipListDestroy(pSkipList); + tfree(total); +} + +// todo not support duplicated key yet +void duplicatedKeyTest() { +#if 0 + SSkipListKey key; + key.nType = TSDB_DATA_TYPE_INT; + + SSkipListNode **pNodes = NULL; + + SSkipList *pSkipList = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_INT, sizeof(int)); + + for (int32_t i = 0; i < 10000; ++i) { + for (int32_t j = 0; j < 5; ++j) { + key.i64Key = i; + tSkipListPut(pSkipList, "", &key, 1); + } + } + + tSkipListPrint(pSkipList, 1); + + for (int32_t i = 0; i < 100; ++i) { + key.i64Key = rand() % 1000; + int32_t size = tSkipListGets(pSkipList, &key, &pNodes); + + assert(size == 5); + + tfree(pNodes); + } + + tSkipListDestroy(pSkipList); +#endif +} + +} // namespace + +TEST(testCase, skiplist_test) { + assert(sizeof(SSkipListKey) == 8); + srand(time(NULL)); + + stringKeySkiplistTest(); + doubleSkipListTest(); + skiplistPerformanceTest(); + duplicatedKeyTest(); + randKeyTest(); + + // tSKipListQueryCond q; + // q.upperBndRelOptr = true; + // q.lowerBndRelOptr = true; + // q.upperBnd.nType = TSDB_DATA_TYPE_DOUBLE; + // q.lowerBnd.nType = TSDB_DATA_TYPE_DOUBLE; + // q.lowerBnd.dKey = 120; + // q.upperBnd.dKey = 171.989; + /* + int32_t size = tSkipListQuery(pSkipList, &q, &pNodes); + for (int32_t i = 0; i < size; ++i) { + printf("-----%lf\n", pNodes[i]->key.dKey); + } + printf("the range query result size is: %d\n", size); + tfree(pNodes); + + SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20); + for (int32_t i = 0; i < 8; i += 2) { + pKeys[i].dKey = i * 0.997; + pKeys[i].nType = TSDB_DATA_TYPE_DOUBLE; + printf("%lf ", pKeys[i].dKey); + } + + int32_t r = tSkipListPointQuery(pSkipList, pKeys, 8, EXCLUDE_POINT_QUERY, &pNodes); + printf("\nthe exclude query result is: %d\n", r); + for (int32_t i = 0; i < r; ++i) { + // printf("%lf ", pNodes[i]->key.dKey); + } + tfree(pNodes); + + free(pKeys);*/ +} diff --git a/src/util/tests/stringTest.cpp b/src/util/tests/stringTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ef8df90e3eac8c4381cbc23cd5311b794b73d058 --- /dev/null +++ b/src/util/tests/stringTest.cpp @@ -0,0 +1,178 @@ +#include +#include +#include +#include + +#include "taos.h" +#include "tutil.h" + +TEST(testCase, string_dequote_test) { + char t1[] = "'abc'"; + int32_t len = strdequote(t1); + + EXPECT_EQ(3, len); + EXPECT_STRCASEEQ(t1, "abc"); + + char t2[] = "\"abc\""; + len = strdequote(t2); + + EXPECT_EQ(3, len); + EXPECT_STRCASEEQ(t1, "abc"); + + char t21[] = " abc "; + strtrim(t21); + + EXPECT_STREQ("abc", t21); + EXPECT_EQ(3, strlen(t21)); +} + +TEST(testCase, string_replace_test) { + char t3[] = "abc01abc02abc"; + char* ret = strreplace(t3, "abc", "7"); + + EXPECT_EQ(strlen(ret), 7); + EXPECT_STREQ("7017027", ret); + free(ret); + + char t4[] = "a01a02b03c04d05"; + ret = strreplace(t4, "0", "9999999999"); + + EXPECT_EQ(strlen(ret), 5 * 10 + 10); + EXPECT_STREQ("a99999999991a99999999992b99999999993c99999999994d99999999995", ret); + free(ret); + + char t5[] = "abc"; + ret = strreplace(t5, "abc", "12345678901234567890"); + + EXPECT_EQ(strlen(ret), 20); + EXPECT_STREQ("12345678901234567890", ret); + free(ret); + + char t6[] = "abc"; + ret = strreplace(t6, "def", "abc"); + + EXPECT_EQ(strlen(ret), 3); + EXPECT_STREQ("abc", ret); + free(ret); + + char t7[] = "abcde000000000000001234"; + ret = strreplace(t7, "ab", "0000000"); + + EXPECT_EQ(strlen(ret), 28); + EXPECT_STREQ("0000000cde000000000000001234", ret); + free(ret); + + char t8[] = "abc\ndef"; + char t[] = {10, 0}; + + char f1[] = "\\n"; + int32_t fx = strlen(f1); + ret = strreplace(t8, "\n", "\\n"); + + EXPECT_EQ(strlen(ret), 8); + EXPECT_STREQ("abc\\ndef", ret); + free(ret); + + char t9[] = "abc\\ndef"; + ret = strreplace(t9, "\\n", "\n"); + + EXPECT_EQ(strlen(ret), 7); + EXPECT_STREQ("abc\ndef", ret); + free(ret); + + char t10[] = "abcdef"; + ret = strreplace(t10, "", "0"); + + EXPECT_EQ(strlen(ret), 6); + EXPECT_STREQ("abcdef", ret); + free(ret); +} + +TEST(testCase, string_tolower_test) { + char t[1024] = {1}; + memset(t, 1, tListLen(t)); + + const char* a1 = "ABC"; + strtolower(t, a1); + EXPECT_STREQ(t, "abc"); + + memset(t, 1, tListLen(t)); + const char* a2 = "ABC\'ABC\'D"; + strtolower(t, a2); + EXPECT_STREQ(t, "abc\'ABC\'d"); + + memset(t, 1, tListLen(t)); + const char* a3 = ""; + strtolower(t, a3); + EXPECT_STREQ(t, ""); + + memset(t, 1, tListLen(t)); + const char* a4 = "\"AbcDEF\""; + strtolower(t, a4); + EXPECT_STREQ(t, a4); + + memset(t, 1, tListLen(t)); + const char* a5 = "1234\"AbcDEF\"456"; + strtolower(t, a5); + EXPECT_STREQ(t, a5); + + memset(t, 1, tListLen(t)); + const char* a6 = "1234"; + strtolower(t, a6); + EXPECT_STREQ(t, a6); +} + +TEST(testCase, string_strnchr_test) { + char t[1024] = {0}; + memset(t, 1, tListLen(t)); + + char a1[] = "AB.C"; + EXPECT_TRUE(strnchr(a1, '.', strlen(a1), true) != NULL); + + char a2[] = "abc."; + EXPECT_TRUE(strnchr(a2, '.', strlen(a2), true) != NULL); + + char a8[] = "abc."; + EXPECT_TRUE(strnchr(a8, '.', 1, true) == NULL); + + char a3[] = ".abc"; + EXPECT_TRUE(strnchr(a3, '.', strlen(a3), true) != NULL); + + char a4[] = "'.abc'"; + EXPECT_TRUE(strnchr(a4, '.', strlen(a4), true) == NULL); + + char a5[] = "'.abc.'abc"; + EXPECT_TRUE(strnchr(a5, '.', strlen(a5), true) == NULL); + + char a6[] = "0123456789."; + EXPECT_TRUE(strnchr(a6, '.', strlen(a6), true) != NULL); + + char a7[] = "0123456789."; + EXPECT_TRUE(strnchr(a7, '.', 3, true) == NULL); + + char a9[] = "0123456789."; + EXPECT_TRUE(strnchr(a9, '.', 0, true) == NULL); + + char a10[] = "0123456789'.'"; + EXPECT_TRUE(strnchr(a10, '.', strlen(a10), true) == NULL); +} + +// TEST(testCase, cache_resize_test) { +// char a11[] = "abc'.'"; +// EXPECT_TRUE(strnchr(a11, '.', strlen(a11), false) != NULL); + +// char a12[] = "abc'-'"; +// EXPECT_TRUE(strnchr(a12, '-', strlen(a12), false) != NULL); + +// char a15[] = "abc'-'"; +// EXPECT_TRUE(strnchr(a15, '-', strlen(a15), true) == NULL); + +// char a13[] = "'-'"; +// EXPECT_TRUE(strnchr(a13, '-', strlen(a13), false) != NULL); + +// char a14[] = "'-'"; +// EXPECT_TRUE(strnchr(a14, '-', strlen(a14), true) == NULL); + +// char a16[] = "'-'."; +// EXPECT_TRUE(strnchr(a16, '.', strlen(a16), true) != NULL); +// } \ No newline at end of file diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index 92d8ad757b864062f459f92f7fcb4aa7fedebb08..ed6bb044c6c4f7c70f01a8ebedef9f034194df93 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -207,11 +207,6 @@ typedef struct SDataBlockInfo { int32_t sid; } SDataBlockInfo; -typedef struct STableIDList { - STableId *tableIds; - int32_t num; -} STableIDList; - typedef struct { } SFields; @@ -325,15 +320,15 @@ tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stable * @param pQueryHandle * @return table sid list. the invoker is responsible for the release of this the sid list. */ -STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle); +SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle); /** - * Get the qualified table sid for a super table according to the tag query expression. + * Get the qualified table id for a super table according to the tag query expression. * @param stableid. super table sid * @param pTagCond. tag query condition * */ -STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond); +SArray *tsdbQueryTableList(struct STsdbRepo* tsdb, int64_t uid, const wchar_t *pTagCond, size_t len); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 9a4d94c58fc99ef8fc647d188d2d41f2979db212..6c42d4aa155addfd1daa8d9d98319dec9bff748d 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -17,13 +17,18 @@ #include +#include "dataformat.h" #include "taosdef.h" #include "tglobalcfg.h" +#include "tsdb.h" #ifdef __cplusplus extern "C" { #endif +#define TSDB_FILE_HEAD_SIZE 512 +#define TSDB_FILE_DELIMITER 0xF00AFA0F + #define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) #define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) @@ -39,13 +44,16 @@ typedef enum { extern const char *tsdbFileSuffix[]; typedef struct { - int8_t type; - int fd; - char fname[128]; int64_t size; // total size of the file int64_t tombSize; // unused file size int32_t totalBlocks; int32_t totalSubBlocks; +} SFileInfo; + +typedef struct { + int fd; + char fname[128]; + SFileInfo info; } SFile; #define TSDB_IS_FILE_OPENED(f) ((f)->fd != -1) @@ -68,21 +76,41 @@ typedef struct { STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles); void tsdbCloseFileH(STsdbFileH *pFileH); +int tsdbCreateFile(char *dataDir, int fileId, char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose); int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); +int tsdbOpenFile(SFile *pFile, int oflag); +int tsdbCloseFile(SFile *pFile); SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid); +#define TSDB_FGROUP_ITER_FORWARD 0 +#define TSDB_FGROUP_ITER_BACKWARD 1 +typedef struct { + int numOfFGroups; + SFileGroup *base; + SFileGroup *pFileGroup; + int direction; +} SFileGroupIter; + +void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction); +void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid); +SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter); + typedef struct { int32_t len; - int32_t padding; // For padding purpose - int64_t offset; -} SCompIdx; + int32_t offset; + int32_t hasLast : 1; + int32_t numOfSuperBlocks : 31; + int32_t checksum; + TSKEY maxKey; +} SCompIdx; /* sizeof(SCompIdx) = 24 */ /** - * if numOfSubBlocks == -1, then the SCompBlock is a sub-block - * if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to - * the data block offset and length - * if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the - * binary + * if numOfSubBlocks == 0, then the SCompBlock is a sub-block + * if numOfSubBlocks >= 1, then the SCompBlock is a super-block + * - if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to + * the data block offset and length + * - if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the + * binary */ typedef struct { int64_t last : 1; // If the block in data file or last file @@ -97,15 +125,27 @@ typedef struct { TSKEY keyLast; } SCompBlock; +#define IS_SUPER_BLOCK(pBlock) ((pBlock)->numOfSubBlocks >= 1) +#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0) + typedef struct { int32_t delimiter; // For recovery usage int32_t checksum; // TODO: decide if checksum logic in this file or make it one API int64_t uid; - int32_t padding; // For padding purpose - int32_t numOfBlocks; // TODO: make the struct padding SCompBlock blocks[]; } SCompInfo; +#define TSDB_COMPBLOCK_AT(pCompInfo, idx) ((pCompInfo)->blocks + (idx)) +#define TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pCompBlock, size)\ +do {\ + if (pCompBlock->numOfSubBlocks > 1) {\ + pCompBlock = pCompInfo->blocks + pCompBlock->offset;\ + size = pCompBlock->numOfSubBlocks;\ + } else {\ + size = 1;\ + }\ +} while (0) + // TODO: take pre-calculation into account typedef struct { int16_t colId; // Column ID @@ -122,6 +162,20 @@ typedef struct { SCompCol cols[]; } SCompData; +STsdbFileH* tsdbGetFile(tsdb_repo_t* pRepo); + +int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols); + +int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables); +int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf); +int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf); +int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf); +int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData); + +SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid); + +// TODO: need an API to merge all sub-block data into one + void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/vnode/tsdb/src/tsdbCache.c index 3496f6a5c8c480bde799adeb04ca4e6668057820..d64b98d49dbcd7c7f76a7fe4c914a975c82c4658 100644 --- a/src/vnode/tsdb/src/tsdbCache.c +++ b/src/vnode/tsdb/src/tsdbCache.c @@ -75,8 +75,10 @@ void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) { if (pCache->curBlock !=NULL && (pCache->mem->list) >= pCache->totalCacheBlocks/2) { tsdbTriggerCommit(pCache->pRepo); } - if (tsdbAllocBlockFromPool(pCache) < 0) { + + while (tsdbAllocBlockFromPool(pCache) < 0) { // TODO: deal with the error + // printf("Failed to allocate from cache pool\n"); } } diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index f622c38b5ff4acf79e5ec405de0343a5d5197a19..d6964112e7fef975aaf0b34c67faefe268fdab4e 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -22,11 +22,9 @@ #include #include +#include "tutil.h" #include "tsdbFile.h" -#define TSDB_FILE_HEAD_SIZE 512 -#define TSDB_FILE_DELIMITER 0xF00AFA0F - const char *tsdbFileSuffix[] = { ".head", // TSDB_FILE_TYPE_HEAD ".data", // TSDB_FILE_TYPE_DATA @@ -35,10 +33,10 @@ const char *tsdbFileSuffix[] = { static int compFGroupKey(const void *key, const void *fgroup); static int compFGroup(const void *arg1, const void *arg2); -static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname); -static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, SFile *pFile); +static int tsdbGetFileName(char *dataDir, int fileId, char *suffix, char *fname); static int tsdbWriteFileHead(SFile *pFile); static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables); +static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid); STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(STsdbFileH) + sizeof(SFileGroup) * maxFiles); @@ -54,10 +52,17 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { return NULL; } - struct dirent *dp; + struct dirent *dp = NULL; + int fid = 0; + SFileGroup fGroup = {0}; while ((dp = readdir(dir)) != NULL) { if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 1) == 0) continue; - // TODO + int fid = 0; + sscanf(dp->d_name, "f%d", &fid); + if (tsdbOpenFGroup(pFileH, dataDir, fid) < 0) { + break; + // TODO + } } return pFileH; @@ -65,17 +70,39 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { void tsdbCloseFileH(STsdbFileH *pFileH) { free(pFileH); } +static int tsdbInitFile(char *dataDir, int fid, char *suffix, SFile *pFile) { + tsdbGetFileName(dataDir, fid, suffix, pFile->fname); + if (access(pFile->fname, F_OK|R_OK|W_OK) < 0) return -1; + pFile->fd = -1; + // TODO: recover the file info + // pFile->info = {0}; + return 0; +} + +static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) { + if (tsdbSearchFGroup(pFileH, fid) != NULL) return 0; + + char fname[128] = "\0"; + SFileGroup fGroup = {0}; + fGroup.fileId = fid; + + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + if (tsdbInitFile(dataDir, fid, tsdbFileSuffix[type], &fGroup.files[type]) < 0) return -1; + } + pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; + qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); + return 0; +} + int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) { if (pFileH->numOfFGroups >= pFileH->maxFGroups) return -1; SFileGroup fGroup; SFileGroup *pFGroup = &fGroup; - if (fid < TSDB_MIN_FILE_ID(pFileH) || fid > TSDB_MAX_FILE_ID(pFileH) || - bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey) == - NULL) { + if (tsdbSearchFGroup(pFileH, fid) == NULL) { // if not exists, create one pFGroup->fileId = fid; for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(dataDir, fid, type, maxTables, &(pFGroup->files[type])) < 0) { + if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]), type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0) { // TODO: deal with the ERROR here, remove those creaed file return -1; } @@ -107,6 +134,143 @@ int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { return 0; } +void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { + pIter->direction = direction; + pIter->base = pFileH->fGroup; + pIter->numOfFGroups = pFileH->numOfFGroups; + if (pFileH->numOfFGroups == 0){ + pIter->pFileGroup = NULL; + } else { + if (direction == TSDB_FGROUP_ITER_FORWARD) { + pIter->pFileGroup = pFileH->fGroup; + } else { + pIter->pFileGroup = pFileH->fGroup + pFileH->numOfFGroups - 1; + } + } +} + +void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { + int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; + void *ptr = taosbsearch(&fid, pIter->base, sizeof(SFileGroup), pIter->numOfFGroups, compFGroupKey, flags); + if (ptr == NULL) { + pIter->pFileGroup = NULL; + } else { + pIter->pFileGroup = (SFileGroup *)ptr; + } +} + +SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { + SFileGroup *ret = pIter->pFileGroup; + if (ret == NULL) return NULL; + + if (pIter->direction = TSDB_FGROUP_ITER_FORWARD) { + if (pIter->pFileGroup + 1 == pIter->base + pIter->numOfFGroups) { + pIter->pFileGroup = NULL; + } else { + pIter->pFileGroup += 1; + } + } else { + if (pIter->pFileGroup - 1 == pIter->base) { + pIter->pFileGroup = NULL; + } else { + pIter->pFileGroup -= 1; + } + } + return ret; +} + +int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) { + SCompBlock *pBlock = pStartBlock; + for (int i = 0; i < numOfBlocks; i++) { + if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1; + for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) { + SCompCol *pCompCol = &(pCompData->cols[iCol]); + pCols->numOfPoints += pBlock->numOfPoints; + int k = 0; + for (; k < pCols->numOfCols; k++) { + if (pCompCol->colId == pCols->cols[k].colId) break; + } + + if (tsdbLoadColData(pFile, pCompCol, pBlock->offset, + (void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0) + return -1; + } + pStartBlock++; + } + return 0; +} + +int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols) { + SCompBlock *pSuperBlock = TSDB_COMPBLOCK_AT(pCompInfo, idx); + SCompBlock *pStartBlock = NULL; + SCompBlock *pBlock = NULL; + int numOfBlocks = pSuperBlock->numOfSubBlocks; + + if (numOfBlocks == 1) + pStartBlock = pSuperBlock; + else + pStartBlock = TSDB_COMPBLOCK_AT(pCompInfo, pSuperBlock->offset); + + int maxNumOfCols = 0; + pBlock = pStartBlock; + for (int i = 0; i < numOfBlocks; i++) { + if (pBlock->numOfCols > maxNumOfCols) maxNumOfCols = pBlock->numOfCols; + pBlock++; + } + + SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * maxNumOfCols); + if (pCompData == NULL) return -1; + + // Load data from the block + if (tsdbLoadDataBlock(pOutFile, pStartBlock, numOfBlocks, pCols, pCompData)); + + // Write data block to the file + { + // TODO + } + + + if (pCompData) free(pCompData); + return 0; +} + +int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) { + SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); + if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + + if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1; + // TODO: need to check the correctness + return 0; +} + +int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) { + SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); + + if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1; + + if (read(pFile->fd, buf, pIdx->len) < 0) return -1; + + // TODO: need to check the correctness + + return 0; +} + +int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf) { + // assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); + + if (lseek(pFile->fd, pBlock->offset, SEEK_SET) < 0) return -1; + size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols; + if (read(pFile->fd, buf, size) < 0) return -1; + + return 0; +} + +int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf) { + if (lseek(pFile->fd, blockBaseOffset + pCol->offset, SEEK_SET) < 0) return -1; + if (read(pFile->fd, buf, pCol->len) < 0) return -1; + return 0; +} + static int compFGroupKey(const void *key, const void *fgroup) { int fid = *(int *)key; SFileGroup *pFGroup = (SFileGroup *)fgroup; @@ -120,7 +284,7 @@ static int compFGroup(const void *arg1, const void *arg2) { static int tsdbWriteFileHead(SFile *pFile) { char head[TSDB_FILE_HEAD_SIZE] = "\0"; - pFile->size += TSDB_FILE_HEAD_SIZE; + pFile->info.size += TSDB_FILE_HEAD_SIZE; // TODO: write version and File statistic to the head lseek(pFile->fd, 0, SEEK_SET); @@ -144,21 +308,21 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) { return -1; } - pFile->size += size; + pFile->info.size += size; free(buf); return 0; } -static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname) { - if (dataDir == NULL || fname == NULL || !IS_VALID_TSDB_FILE_TYPE(type)) return -1; +static int tsdbGetFileName(char *dataDir, int fileId, char *suffix, char *fname) { + if (dataDir == NULL || fname == NULL) return -1; - sprintf(fname, "%s/f%d%s", dataDir, fileId, tsdbFileSuffix[type]); + sprintf(fname, "%s/f%d%s", dataDir, fileId, suffix); return 0; } -static int tsdbOpenFileForWrite(SFile *pFile, int oflag) { // TODO: change the function +int tsdbOpenFile(SFile *pFile, int oflag) { // TODO: change the function if (TSDB_IS_FILE_OPENED(pFile)) return -1; pFile->fd = open(pFile->fname, oflag, 0755); @@ -167,31 +331,39 @@ static int tsdbOpenFileForWrite(SFile *pFile, int oflag) { // TODO: change the f return 0; } -static int tsdbCloseFile(SFile *pFile) { - if (!TSDB_IS_FILE_OPENED(pFile)) return -1; +int tsdbCloseFile(SFile *pFile) { int ret = close(pFile->fd); pFile->fd = -1; - return ret; } -static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, SFile *pFile) { +SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) { + SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid); + if (pGroup == NULL) return NULL; + + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + tsdbOpenFile(&(pGroup->files[type]), O_RDWR); + } + return pGroup; +} + +int tsdbCreateFile(char *dataDir, int fileId, char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose) { memset((void *)pFile, 0, sizeof(SFile)); - pFile->type = type; pFile->fd = -1; - tsdbGetFileName(dataDir, fileId, type, pFile->fname); + tsdbGetFileName(dataDir, fileId, suffix, pFile->fname); + if (access(pFile->fname, F_OK) == 0) { // File already exists return -1; } - if (tsdbOpenFileForWrite(pFile, O_WRONLY | O_CREAT) < 0) { + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) { // TODO: deal with the ERROR here return -1; } - if (type == TSDB_FILE_TYPE_HEAD) { + if (writeHeader) { if (tsdbWriteHeadFileIdx(pFile, maxTables) < 0) { tsdbCloseFile(pFile); return -1; @@ -203,7 +375,7 @@ static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, return -1; } - tsdbCloseFile(pFile); + if (toClose) tsdbCloseFile(pFile); return 0; } @@ -212,4 +384,12 @@ void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t file TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; +} + +SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid) { + if (pFileH->numOfFGroups == 0 || fid < pFileH->fGroup[0].fileId || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) + return NULL; + void *ptr = bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey); + if (ptr == NULL) return NULL; + return (SFileGroup *)ptr; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index af3a923d904d62c5fc3af8d8f4ca8dfad64c7282..6d675d300a418b521fd5595396c25f3dfac88d55 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -8,6 +8,7 @@ #include #include #include +#include #include // #include "taosdef.h" @@ -45,6 +46,7 @@ #define TSDB_CFG_FILE_NAME "CONFIG" #define TSDB_DATA_DIR_NAME "data" #define TSDB_DEFAULT_FILE_BLOCK_ROW_OPTION 0.7 +#define TSDB_MAX_LAST_FILE_SIZE (1024 * 1024 * 10) // 10M enum { TSDB_REPO_STATE_ACTIVE, TSDB_REPO_STATE_CLOSED, TSDB_REPO_STATE_CONFIGURING }; @@ -84,7 +86,12 @@ static int tsdbOpenMetaFile(char *tsdbDir); static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); -static void * tsdbCommitToFile(void *arg); +static void * tsdbCommitData(void *arg); +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols); +static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey); +static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey); +static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, + int64_t uid); #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid] #define TSDB_GET_TABLE_BY_NAME(pRepo, name) @@ -230,6 +237,7 @@ int32_t tsdbDropRepo(tsdb_repo_t *repo) { * @return a TSDB repository handle on success, NULL for failure and the error number is set */ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) { + char dataDir[128] = "\0"; if (access(tsdbDir, F_OK | W_OK | R_OK) < 0) { return NULL; } @@ -258,6 +266,16 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) { return NULL; } + tsdbGetDataDirName(pRepo, dataDir); + pRepo->tsdbFileH = tsdbInitFileH(dataDir, pRepo->config.maxTables); + if (pRepo->tsdbFileH == NULL) { + tsdbFreeCache(pRepo->tsdbCache); + tsdbFreeMeta(pRepo->tsdbMeta); + free(pRepo->rootDir); + free(pRepo); + return NULL; + } + pRepo->state = TSDB_REPO_STATE_ACTIVE; return (tsdb_repo_t *)pRepo; @@ -280,8 +298,29 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo) { if (pRepo == NULL) return 0; pRepo->state = TSDB_REPO_STATE_CLOSED; + tsdbLockRepo(repo); + if (pRepo->commit) { + tsdbUnLockRepo(repo); + return -1; + } + pRepo->commit = 1; + // Loop to move pData to iData + for (int i = 0; i < pRepo->config.maxTables; i++) { + STable *pTable = pRepo->tsdbMeta->tables[i]; + if (pTable != NULL && pTable->mem != NULL) { + pTable->imem = pTable->mem; + pTable->mem = NULL; + } + } + // TODO: Loop to move mem to imem + pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; + pRepo->tsdbCache->mem = NULL; + pRepo->tsdbCache->curBlock = NULL; + tsdbUnLockRepo(repo); + + tsdbCommitData((void *)repo); - tsdbFlushCache(pRepo); + tsdbCloseFileH(pRepo->tsdbFileH); tsdbFreeMeta(pRepo->tsdbMeta); @@ -325,22 +364,25 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; pRepo->tsdbCache->mem = NULL; pRepo->tsdbCache->curBlock = NULL; + tsdbUnLockRepo(repo); // TODO: here should set as detached or use join for memory leak - pthread_create(&(pRepo->commitThread), NULL, tsdbCommitToFile, (void *)repo); - tsdbUnLockRepo(repo); + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); + pthread_create(&(pRepo->commitThread), &thattr, tsdbCommitData, (void *)repo); return 0; } int32_t tsdbLockRepo(tsdb_repo_t *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; - return pthread_mutex_lock(repo); + return pthread_mutex_lock(&(pRepo->mutex)); } int32_t tsdbUnLockRepo(tsdb_repo_t *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; - return pthread_mutex_unlock(repo); + return pthread_mutex_unlock(&(pRepo->mutex)); } /** @@ -541,6 +583,11 @@ STsdbMeta* tsdbGetMeta(tsdb_repo_t* pRepo) { return tsdb->tsdbMeta; } +STsdbFileH* tsdbGetFile(tsdb_repo_t* pRepo) { + STsdbRepo* tsdb = (STsdbRepo*) pRepo; + return tsdb->tsdbFileH; +} + // Check the configuration and set default options static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { // Check precision @@ -704,7 +751,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable if (pTable->mem == NULL) { pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); if (pTable->mem == NULL) return -1; - pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); + pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey); pTable->mem->keyFirst = INT64_MAX; pTable->mem->keyLast = 0; } @@ -727,7 +774,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable if (pTable->mem == NULL) { pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); if (pTable->mem == NULL) return -1; - pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); + pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey); pTable->mem->keyFirst = INT64_MAX; pTable->mem->keyLast = 0; } @@ -761,24 +808,22 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { return 0; } -static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCol **cols, STSchema *pSchema) { +static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { int numOfRows = 0; + do { SSkipListNode *node = tSkipListIterGet(pIter); if (node == NULL) break; SDataRow row = SL_GET_NODE_DATA(node); if (dataRowKey(row) > maxKey) break; - // Convert row data to column data - // for (int i = 0; i < schemaNCols(pSchema); i++) { - // STColumn *pCol = schemaColAt(pSchema, i); - // memcpy(cols[i]->data + TYPE_BYTES[colType(pCol)] * numOfRows, dataRowAt(row, pCol->offset), - // TYPE_BYTES[colType(pCol)]); - // } + + tdAppendDataRowToDataCol(row, pCols); numOfRows++; - if (numOfRows > maxRowsToRead) break; + if (numOfRows >= maxRowsToRead) break; } while (tSkipListIterNext(pIter)); + return numOfRows; } @@ -808,7 +853,9 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) } if (!tSkipListIterNext(iters[tid])) { - assert(false); + // No data in this iterator + tSkipListDestroyIter(iters[tid]); + iters[tid] = NULL; } } @@ -816,14 +863,14 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) } // Commit to file -static void *tsdbCommitToFile(void *arg) { +static void *tsdbCommitData(void *arg) { // TODO printf("Starting to commit....\n"); STsdbRepo * pRepo = (STsdbRepo *)arg; STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbCache *pCache = pRepo->tsdbCache; STsdbCfg * pCfg = &(pRepo->config); - if (pCache->imem == NULL) return; + if (pCache->imem == NULL) return NULL; // Create the iterator to read from cache SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables); @@ -832,53 +879,26 @@ static void *tsdbCommitToFile(void *arg) { return NULL; } - int maxCols = pMeta->maxCols; - int maxBytes = pMeta->maxRowBytes; - SDataCol **cols = (SDataCol **)malloc(sizeof(SDataCol *) * maxCols); - void *buf = malloc((maxBytes + sizeof(SDataCol)) * pCfg->maxRowsPerFileBlock); + // Create a data column buffer for commit + SDataCols *pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock); + if (pDataCols == NULL) { + // TODO: deal with the error + return NULL; + } int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); for (int fid = sfid; fid <= efid; fid++) { - TSKEY minKey = 0, maxKey = 0; - tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); - - // tsdbOpenFileForWrite(pRepo, fid); - - for (int tid = 0; tid < pCfg->maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; - if (pTable == NULL || pTable->imem == NULL) continue; - if (iters[tid] == NULL) { // create table iterator - iters[tid] = tSkipListCreateIter(pTable->imem->pData); - // TODO: deal with the error - if (iters[tid] == NULL) break; - if (!tSkipListIterNext(iters[tid])) { - // assert(0); - } - } - - // Init row data part - cols[0] = (SDataCol *)buf; - for (int col = 1; col < schemaNCols(pTable->schema); col++) { - cols[col] = (SDataCol *)((char *)(cols[col - 1]) + sizeof(SDataCol) + colBytes(schemaColAt(pTable->schema, col-1)) * pCfg->maxRowsPerFileBlock); - } - - // Loop the iterator - int rowsRead = 0; - while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, cols, pTable->schema)) > - 0) { - // printf("rowsRead:%d-----------\n", rowsRead); - int k = 0; - } + if (tsdbCommitToFile(pRepo, fid, iters, pDataCols) < 0) { + // TODO: deal with the error here + // assert(0); } } + tdFreeDataCols(pDataCols); tsdbDestroyTableIters(iters, pCfg->maxTables); - free(buf); - free(cols); - tsdbLockRepo(arg); tdListMove(pCache->imem->list, pCache->pool.memPool); free(pCache->imem); @@ -894,4 +914,352 @@ static void *tsdbCommitToFile(void *arg) { tsdbUnLockRepo(arg); return NULL; +} + +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) { + int isNewLastFile = 0; + + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + STsdbCfg * pCfg = &pRepo->config; + SFile hFile, lFile; + SFileGroup *pGroup = NULL; + SCompIdx * pIndices = NULL; + SCompInfo * pCompInfo = NULL; + size_t compInfoSize = 0; + SCompBlock compBlock; + SCompBlock *pBlock = &compBlock; + + TSKEY minKey = 0, maxKey = 0; + tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + + // Check if there are data to commit to this file + int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); + if (!hasDataToCommit) return 0; // No data to commit, just return + + // TODO: make it more flexible + pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + sizeof(SCompBlock) * 1000); + + // Create and open files for commit + tsdbGetDataDirName(pRepo, dataDir); + if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { /* TODO */ + } + pGroup = tsdbOpenFilesForCommit(pFileH, fid); + if (pGroup == NULL) { /* TODO */ + } + tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &hFile, 1, 0); + if (0 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { + // TODO: make it not to write the last file every time + tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0); + isNewLastFile = 1; + } + + // Load the SCompIdx + pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); + if (pIndices == NULL) { /* TODO*/ + } + if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { /* TODO */ + } + + lseek(hFile.fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, SEEK_SET); + + // Loop to commit data in each table + for (int tid = 0; tid < pCfg->maxTables; tid++) { + STable * pTable = pMeta->tables[tid]; + SSkipListIterator *pIter = iters[tid]; + SCompIdx * pIdx = &pIndices[tid]; + + int nNewBlocks = 0; + + if (pTable == NULL || pIter == NULL) continue; + + /* If no new data to write for this table, just write the old data to new file + * if there are. + */ + if (!tsdbHasDataInRange(pIter, minKey, maxKey)) { + // has old data + if (pIdx->len > 0) { + goto _table_over; + // if (isNewLastFile && pIdx->hasLast) { + if (0) { + // need to move the last block to new file + if ((pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len)) == NULL) { /* TODO */ + } + if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */ + } + + tdInitDataCols(pCols, pTable->schema); + + SCompBlock *pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks); + int nBlocks = 0; + + TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pTBlock, nBlocks); + + SCompBlock tBlock; + int64_t toffset, tlen; + tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_LAST], pTBlock, nBlocks, pCols, &tBlock); + + tsdbWriteBlockToFileImpl(&lFile, pCols, pCols->numOfPoints, &toffset, tlen, pTable->tableId.uid); + pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks); + pTBlock->offset = toffset; + pTBlock->len = tlen; + pTBlock->numOfPoints = pCols->numOfPoints; + pTBlock->numOfSubBlocks = 1; + + pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); + if (nBlocks > 1) { + pIdx->len -= (sizeof(SCompBlock) * nBlocks); + } + write(hFile.fd, (void *)pCompInfo, pIdx->len); + } else { + pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); + sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len); + hFile.info.size += pIdx->len; + } + } + continue; + } + + pCompInfo->delimiter = TSDB_FILE_DELIMITER; + pCompInfo->checksum = 0; + pCompInfo->uid = pTable->tableId.uid; + + // Load SCompBlock part if neccessary + int isCompBlockLoaded = 0; + if (0) { + // if (pIdx->offset > 0) { + if (pIdx->hasLast || tsdbHasDataInRange(pIter, minKey, pIdx->maxKey)) { + // has last block || cache key overlap with commit key + pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len + sizeof(SCompBlock) * 100); + if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */ + } + if (pCompInfo->uid == pTable->tableId.uid) isCompBlockLoaded = 1; + } else { + // TODO: No need to load the SCompBlock part, just sendfile the SCompBlock part + // and write those new blocks to it + } + } + + tdInitDataCols(pCols, pTable->schema); + + int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; + while (1) { + tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols); + if (pCols->numOfPoints == 0) break; + + int pointsWritten = pCols->numOfPoints; + // TODO: all write to the end of .data file + int64_t toffset = 0; + int32_t tlen = 0; + tsdbWriteBlockToFileImpl(&pGroup->files[TSDB_FILE_TYPE_DATA], pCols, pCols->numOfPoints, &toffset, &tlen, pTable->tableId.uid); + + // Make the compBlock + SCompBlock *pTBlock = pCompInfo->blocks + nNewBlocks++; + pTBlock->offset = toffset; + pTBlock->len = tlen; + pTBlock->keyFirst = dataColsKeyFirst(pCols); + pTBlock->keyLast = dataColsKeyLast(pCols); + pTBlock->last = 0; + pTBlock->algorithm = 0; + pTBlock->numOfPoints = pCols->numOfPoints; + pTBlock->sversion = pTable->sversion; + pTBlock->numOfSubBlocks = 1; + + if (dataColsKeyLast(pCols) > pIdx->maxKey) pIdx->maxKey = dataColsKeyLast(pCols); + + tdPopDataColsPoints(pCols, pointsWritten); + maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pCols->numOfPoints; + } + + +_table_over: + // Write the SCompBlock part + pIdx->offset = lseek(hFile.fd, 0, SEEK_END); + if (pIdx->len > 0) { + sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len); + if (nNewBlocks > 0) { + write(hFile.fd, (void *)(pCompInfo->blocks), sizeof(SCompBlock) * nNewBlocks); + pIdx->len += (sizeof(SCompBlock) * nNewBlocks); + } + } else { + if (nNewBlocks > 0) { + write(hFile.fd, (void *)pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks); + pIdx->len += sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks; + } + } + + pIdx->checksum = 0; + pIdx->numOfSuperBlocks += nNewBlocks; + pIdx->hasLast = 0; + } + + // Write the SCompIdx part + if (lseek(hFile.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {/* TODO */} + if (write(hFile.fd, (void *)pIndices, sizeof(SCompIdx) * pCfg->maxTables) < 0) {/* TODO */} + + // close the files + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + tsdbCloseFile(&pGroup->files[type]); + } + tsdbCloseFile(&hFile); + if (isNewLastFile) tsdbCloseFile(&lFile); + // TODO: replace the .head and .last file + rename(hFile.fname, pGroup->files[TSDB_FILE_TYPE_HEAD].fname); + pGroup->files[TSDB_FILE_TYPE_HEAD].info = hFile.info; + if (isNewLastFile) { + rename(lFile.fname, pGroup->files[TSDB_FILE_TYPE_LAST].fname); + pGroup->files[TSDB_FILE_TYPE_LAST].info = lFile.info; + } + + if (pIndices) free(pIndices); + if (pCompInfo) free(pCompInfo); + + return 0; +} + +static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey) { + if (pIter == NULL) return 0; + + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) return 0; + + SDataRow row = SL_GET_NODE_DATA(node); + if (dataRowKey(row) >= minKey && dataRowKey(row) <= maxKey) return 1; + + return 0; +} + +static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) { + for (int i = 0; i < nIters; i++) { + SSkipListIterator *pIter = iters[i]; + if (tsdbHasDataInRange(pIter, minKey, maxKey)) return 1; + } + return 0; +} + +static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, int64_t uid) { + size_t size = sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols; + SCompData *pCompData = (SCompData *)malloc(size); + if (pCompData == NULL) return -1; + + pCompData->delimiter = TSDB_FILE_DELIMITER; + pCompData->uid = uid; + pCompData->numOfCols = pCols->numOfCols; + + *offset = lseek(pFile->fd, 0, SEEK_END); + *len = size; + + int toffset = size; + for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { + SCompCol *pCompCol = pCompData->cols + iCol; + SDataCol *pDataCol = pCols->cols + iCol; + + pCompCol->colId = pDataCol->colId; + pCompCol->type = pDataCol->type; + pCompCol->offset = toffset; + + // TODO: add compression + pCompCol->len = TYPE_BYTES[pCompCol->type] * pointsToWrite; + toffset += pCompCol->len; + } + + // Write the block + if (write(pFile->fd, (void *)pCompData, size) < 0) goto _err; + *len += size; + for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { + SDataCol *pDataCol = pCols->cols + iCol; + SCompCol *pCompCol = pCompData->cols + iCol; + if (write(pFile->fd, pDataCol->pData, pCompCol->len) < 0) goto _err; + *len += pCompCol->len; + } + + if (pCompData == NULL) free((void *)pCompData); + return 0; + +_err: + if (pCompData == NULL) free((void *)pCompData); + return -1; +} + +static int compareKeyBlock(const void *arg1, const void *arg2) { + TSKEY key = *(TSKEY *)arg1; + SCompBlock *pBlock = (SCompBlock *)arg2; + + if (key < pBlock->keyFirst) { + return -1; + } else if (key > pBlock->keyLast) { + return 1; + } + + return 0; +} + +int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) { + STsdbCfg * pCfg = &(pRepo->config); + SCompData *pCompData = NULL; + SFile * pFile = NULL; + int numOfPointsToWrite = 0; + int64_t offset = 0; + int32_t len = 0; + + memset((void *)pCompBlock, 0, sizeof(SCompBlock)); + + if (pCompInfo == NULL) { + // Just append the data block to .data or .l or .last file + numOfPointsToWrite = pCols->numOfPoints; + if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // Write to .data file + pFile = &(pGroup->files[TSDB_FILE_TYPE_DATA]); + } else { // Write to .last or .l file + pCompBlock->last = 1; + if (lFile) { + pFile = lFile; + } else { + pFile = &(pGroup->files[TSDB_FILE_TYPE_LAST]); + } + } + tsdbWriteBlockToFileImpl(pFile, pCols, numOfPointsToWrite, &offset, &len, uid); + pCompBlock->offset = offset; + pCompBlock->len = len; + pCompBlock->algorithm = 2; // TODO : add to configuration + pCompBlock->sversion = pCols->sversion; + pCompBlock->numOfPoints = pCols->numOfPoints; + pCompBlock->numOfSubBlocks = 1; + pCompBlock->numOfCols = pCols->numOfCols; + pCompBlock->keyFirst = dataColsKeyFirst(pCols); + pCompBlock->keyLast = dataColsKeyLast(pCols); + } else { + // Need to merge the block to either the last block or the other block + TSKEY keyFirst = dataColsKeyFirst(pCols); + SCompBlock *pMergeBlock = NULL; + + // Search the block to merge in + void *ptr = taosbsearch((void *)&keyFirst, (void *)(pCompInfo->blocks), sizeof(SCompBlock), pIdx->numOfSuperBlocks, + compareKeyBlock, TD_GE); + if (ptr == NULL) { + // No block greater or equal than the key, but there are data in the .last file, need to merge the last file block + // and merge the data + pMergeBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks - 1); + } else { + pMergeBlock = (SCompBlock *)ptr; + } + + if (pMergeBlock->last) { + if (pMergeBlock->last + pCols->numOfPoints > pCfg->minRowsPerFileBlock) { + // Need to load the data from .last and combine data in pCols to write to .data file + + } else { // Just append the block to .last or .l file + if (lFile) { + // read the block from .last file and merge with pCols, write to .l file + + } else { + // tsdbWriteBlockToFileImpl(); + } + } + } else { // The block need to merge in .data file + + } + + } + + return numOfPointsToWrite; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 72c1667fe1862b49c1d351f33e78e59ba597afef..22680a839b28db0d26723df94eedcb58a1758abe 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -102,7 +102,7 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { if (pTable->type == TSDB_SUPER_TABLE) { pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, getTupleKey); + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, 0, getTupleKey); } tsdbAddTableToMeta(pMeta, pTable, false); @@ -207,7 +207,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { super->tagSchema = tdDupSchema(pCfg->tagSchema); super->tagVal = tdDataRowDup(pCfg->tagValues); super->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, - 0, getTupleKey); // Allow duplicate key, no lock + 0, 0, getTupleKey); // Allow duplicate key, no lock if (super->pIndex == NULL) { tdFreeSchema(super->schema); @@ -349,10 +349,10 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) { } else { // add non-super table to the array pMeta->tables[pTable->tableId.tid] = pTable; - if (pTable->type == TSDB_CHILD_TABLE) { - // add STABLE to the index + if (pTable->type == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index tsdbAddTableIntoIndex(pMeta, pTable); } + pMeta->nTables++; } @@ -373,8 +373,27 @@ static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable) { return 0; } static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { - assert(pTable->type == TSDB_CHILD_TABLE); - // TODO + assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); + STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); + assert(pSTable != NULL); + + int32_t level = 0; + int32_t headSize = 0; + + // first tag column + STColumn* s = schemaColAt(pSTable->tagSchema, 0); + + tSkipListRandNodeInfo(pSTable->pIndex, &level, &headSize); + SSkipListNode* pNode = calloc(1, headSize + s->bytes + POINTER_BYTES); + pNode->level = level; + + SSkipList* list = pSTable->pIndex; + + memcpy(SL_GET_NODE_KEY(list, pNode), dataRowTuple(pTable->tagVal), colBytes(s)); + memcpy(SL_GET_NODE_DATA(pNode), &pTable, POINTER_BYTES); + + tSkipListPut(list, pNode); + return 0; } diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 2dc61f51072fac850d697cfad57a6446c7a3eace..2919b2cf9e9229ecc7bf558cb484833d5dc2cb28 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -14,8 +14,12 @@ */ #include "os.h" + +#include "tlog.h" #include "tutil.h" +#include "../../../query/inc/qast.h" +#include "../../../query/inc/tsqlfunction.h" #include "tsdb.h" #include "tsdbFile.h" #include "tsdbMeta.h" @@ -25,6 +29,11 @@ #define QUERY_IS_ASC_QUERY(o) (o == TSQL_SO_ASC) #define QH_GET_NUM_OF_COLS(handle) (taosArrayGetSize((handle)->pColumns)) +enum { + QUERY_RANGE_LESS_EQUAL = 0, + QUERY_RANGE_GREATER_EQUAL = 1, +}; + typedef struct SField { // todo need the definition } SField; @@ -33,12 +42,12 @@ typedef struct SHeaderFileInfo { int32_t fileId; } SHeaderFileInfo; -typedef struct SQueryHandlePos { - int32_t fileId; +typedef struct SQueryFilePos { + int32_t fid; int32_t slot; int32_t pos; - int32_t fileIndex; -} SQueryHandlePos; + int64_t lastKey; +} SQueryFilePos; typedef struct SDataBlockLoadInfo { int32_t fileListIndex; @@ -70,14 +79,20 @@ typedef struct SQueryFilesInfo { char dbFilePathPrefix[PATH_MAX]; } SQueryFilesInfo; -typedef struct STableQueryRec { +typedef struct STableCheckInfo { + STableId tableId; TSKEY lastKey; STable * pTableObj; int64_t offsetInHeaderFile; - int32_t numOfBlocks; +// int32_t numOfBlocks; int32_t start; + bool checkFirstFileBlock; + + SCompIdx* compIndex; + SCompBlock *pBlock; -} STableQueryRec; + SSkipListIterator* iter; +} STableCheckInfo; typedef struct { SCompBlock *compBlock; @@ -86,23 +101,27 @@ typedef struct { typedef struct STableDataBlockInfoEx { SCompBlockFields pBlock; - STableQueryRec * pMeterDataInfo; + STableCheckInfo* pMeterDataInfo; int32_t blockIndex; int32_t groupIdx; /* number of group is less than the total number of meters */ } STableDataBlockInfoEx; +enum { + SINGLE_TABLE_MODEL = 1, + MULTI_TABLE_MODEL = 2, +}; + typedef struct STsdbQueryHandle { struct STsdbRepo* pTsdb; - - SQueryHandlePos cur; // current position - SQueryHandlePos start; // the start position, used for secondary/third iteration + int8_t model; // access model, single table model or multi-table model + SQueryFilePos cur; // current position + SQueryFilePos start; // the start position, used for secondary/third iteration int32_t unzipBufSize; - char *unzipBuffer; - char *secondaryUnzipBuffer; + char *unzipBuffer; + char *secondaryUnzipBuffer; SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ - SQueryFilesInfo vnodeFileInfo; int16_t numOfRowsPerPage; @@ -110,21 +129,21 @@ typedef struct STsdbQueryHandle { int16_t order; STimeWindow window; // the primary query time window that applies to all queries int32_t blockBufferSize; - SCompBlock *pBlock; + SCompBlock* pBlock; int32_t numOfBlocks; SField ** pFields; SArray * pColumns; // column list, SColumnInfoEx array list - SArray * pTableIdList; // table id object list bool locateStart; int32_t realNumOfRows; bool loadDataAfterSeek; // load data after seek. - + SArray* pTableCheckInfo; + int32_t activeIndex; + + int32_t tableIndex; + bool isFirstSlot; + void * qinfo; // query info handle, for debug purpose + STableDataBlockInfoEx *pDataBlockInfoEx; - STableQueryRec * pTableQueryInfo; - int32_t tableIndex; - bool isFirstSlot; - void * qinfo; // query info handle, for debug purpose - SSkipListIterator* memIter; } STsdbQueryHandle; int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) { @@ -260,24 +279,29 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; - pQueryHandle->pTableIdList = idList; pQueryHandle->pColumns = pColumnInfo; pQueryHandle->loadDataAfterSeek = false; pQueryHandle->isFirstSlot = true; - // only support table query - assert(taosArrayGetSize(idList) == 1); - - pQueryHandle->pTableQueryInfo = calloc(1, sizeof(STableQueryRec)); - STableQueryRec* pTableQRec = pQueryHandle->pTableQueryInfo; - - pTableQRec->lastKey = pQueryHandle->window.skey; + size_t size = taosArrayGetSize(idList); + assert(size >= 1); + + pQueryHandle->pTableCheckInfo = taosArrayInit(size, sizeof(STableCheckInfo)); + for(int32_t i = 0; i < size; ++i) { + STableId id = *(STableId*) taosArrayGet(idList, i); + + STableCheckInfo info = { + .lastKey = pQueryHandle->window.skey, + .tableId = id, + .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), //todo this may be failed + }; + + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + } - STableIdInfo* idInfo = taosArrayGet(pQueryHandle->pTableIdList, 0); + pQueryHandle->model = (size > 1)? MULTI_TABLE_MODEL:SINGLE_TABLE_MODEL; - STableId tableId = {.uid = idInfo->uid, .tid = idInfo->sid}; - STable *pTable = tsdbIsValidTableToInsert(tsdbGetMeta(pQueryHandle->pTsdb), tableId); - pTableQRec->pTableObj = pTable; + pQueryHandle->activeIndex = 0; // malloc buffer in order to load data from file int32_t numOfCols = taosArrayGetSize(pColumnInfo); @@ -307,9 +331,13 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond return (tsdb_query_handle_t)pQueryHandle; } -bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - STable *pTable = pHandle->pTableQueryInfo->pTableObj; +static bool hasMoreDataInCacheForSingleModel(STsdbQueryHandle* pHandle) { + assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); + + STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + STable *pTable = pTableCheckInfo->pTableObj; + assert(pTable != NULL); // no data in cache, abort if (pTable->mem == NULL && pTable->imem == NULL) { @@ -317,13 +345,380 @@ bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { } // all data in mem are checked already. - if (pHandle->pTableQueryInfo->lastKey > pTable->mem->keyLast) { + if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { return false; } return true; } +// todo dynamic get the daysperfile +static int32_t getFileIdFromKey(TSKEY key) { + return (int32_t)(key / 10); // set the starting fileId +} + +static int32_t getFileCompInfo(STableCheckInfo* pCheckInfo, SFileGroup* fileGroup) { + tsdbLoadCompIdx(fileGroup, pCheckInfo->compIndex, 10000); // todo set dynamic max tables + SCompIdx* compIndex = &pCheckInfo->compIndex[pCheckInfo->tableId.tid]; + + if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file + + } else { + tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pBlock); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t binarySearchForBlockImpl(SCompBlock *pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { + int32_t firstSlot = 0; + int32_t lastSlot = numOfBlocks - 1; + + int32_t midSlot = firstSlot; + + while (1) { + numOfBlocks = lastSlot - firstSlot + 1; + midSlot = (firstSlot + (numOfBlocks >> 1)); + + if (numOfBlocks == 1) break; + + if (skey > pBlock[midSlot].keyLast) { + if (numOfBlocks == 2) break; + if ((order == TSQL_SO_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; + firstSlot = midSlot + 1; + } else if (skey < pBlock[midSlot].keyFirst) { + if ((order == TSQL_SO_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; + lastSlot = midSlot - 1; + } else { + break; // got the slot + } + } + + return midSlot; +} + +static SDataBlockInfo getTrueBlockInfo(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { + SDataBlockInfo info = {{0}, 0}; + + SCompBlock *pDiskBlock = &pCheckInfo->pBlock[pHandle->cur.slot]; + + info.window.skey = pDiskBlock->keyFirst; + info.window.ekey = pDiskBlock->keyLast; + info.size = pDiskBlock->numOfPoints; + info.numOfCols = pDiskBlock->numOfCols; + + return info; +} + +bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { + SQueryFilePos *cur = &pQueryHandle->cur; + + if (pQueryHandle->cur.fid >= 0) { + int32_t fileIndex = -1; + + /* + * 1. ascending order. The last data block of data file + * 2. descending order. The first block of file + */ + if ((step == QUERY_ASC_FORWARD_STEP && (pQueryHandle->cur.slot == pQueryHandle->numOfBlocks - 1)) || + (step == QUERY_DESC_FORWARD_STEP && (pQueryHandle->cur.slot == 0))) { + // temporarily keep the position value, in case of no data qualified when move forwards(backwards) + SQueryFilePos save = pQueryHandle->cur; + +// fileIndex = getNextDataFileCompInfo(pQueryHandle, &pQueryHandle->cur, &pQueryHandle->vnodeFileInfo, step); + + // first data block in the next file + if (fileIndex >= 0) { + cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; + cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; +// return loadQaulifiedData(pQueryHandle); + } else {// try data in cache + assert(cur->fid == -1); + + if (step == QUERY_ASC_FORWARD_STEP) { +// TSKEY nextTimestamp = +// getQueryStartPositionInCache_rv(pQueryHandle, &pQueryHandle->cur.slot, &pQueryHandle->cur.pos, true); +// if (nextTimestamp < 0) { +// pQueryHandle->cur = save; +// } +// +// return (nextTimestamp > 0); + } + + // no data to check for desc order query, restore the saved position value + pQueryHandle->cur = save; + return false; + } + } + + // next block in the same file + int32_t fid = cur->fid; +// fileIndex = vnodeGetVnodeHeaderFileIndex(&fid, pQueryHandle->order, &pQueryHandle->vnodeFileInfo); + cur->slot += step; + + SCompBlock *pBlock = &pQueryHandle->pBlock[cur->slot]; + cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; +// return loadQaulifiedData(pQueryHandle); + } else { // data in cache + return hasMoreDataInCacheForSingleModel(pQueryHandle); + } +} + +int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { + int firstPos, lastPos, midPos = -1; + int numOfPoints; + TSKEY *keyList; + + if (num <= 0) return -1; + + keyList = (TSKEY *)pValue; + firstPos = 0; + lastPos = num - 1; + + if (order == 0) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[lastPos]) return lastPos; + if (key == keyList[firstPos]) return firstPos; + if (key < keyList[firstPos]) return firstPos - 1; + + numOfPoints = lastPos - firstPos + 1; + midPos = (numOfPoints >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfPoints = lastPos - firstPos + 1; + midPos = (numOfPoints >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + +static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SArray *sa) { + // only return the qualified data to client in terms of query time window, data rows in the same block but do not + // be included in the query time window will be discarded + SQueryFilePos *cur = &pQueryHandle->cur; + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + SDataBlockInfo blockInfo = getTrueBlockInfo(pQueryHandle, pCheckInfo); + + int32_t endPos = cur->pos; + if (QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { + endPos = blockInfo.size - 1; + pQueryHandle->realNumOfRows = endPos - cur->pos + 1; + } else if (!QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { + endPos = 0; + pQueryHandle->realNumOfRows = cur->pos + 1; + } else { +// endPos = vnodeBinarySearchKey(pQueryHandle->tsBuf->data, blockInfo.size, pQueryHandle->window.ekey, pQueryHandle->order); + + if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (endPos < cur->pos) { + pQueryHandle->realNumOfRows = 0; + return; + } else { + pQueryHandle->realNumOfRows = endPos - cur->pos; + } + } else { + if (endPos > cur->pos) { + pQueryHandle->realNumOfRows = 0; + return; + } else { + pQueryHandle->realNumOfRows = cur->pos - endPos; + } + } + } + + int32_t start = MIN(cur->pos, endPos); + + // move the data block in the front to data block if needed + if (start != 0) { + int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + + for (int32_t i = 0; i < taosArrayGetSize(sa); ++i) { + int16_t colId = *(int16_t *)taosArrayGet(sa, i); + + for (int32_t j = 0; j < numOfCols; ++j) { + SColumnInfoEx *pCol = taosArrayGet(pQueryHandle->pColumns, j); + + if (pCol->info.colId == colId) { + memmove(pCol->pData, ((char *)pCol->pData) + pCol->info.bytes * start, pQueryHandle->realNumOfRows * pCol->info.bytes); + break; + } + } + } + } + + assert(pQueryHandle->realNumOfRows <= blockInfo.size); + + // forward(backward) the position for cursor + cur->pos = endPos; +} + +static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { + STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); + int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); + + SFileGroup* fileGroup = tsdbSearchFGroup(pFileHandle, fid); + if (fileGroup == NULL) { + return false; + } + + SQueryFilePos* cur = &pQueryHandle->cur; + + TSKEY key = pCheckInfo->lastKey; + int32_t index = -1; + + // todo add iterator for filegroup + while (1) { + if ((fid = getFileCompInfo(pCheckInfo, fileGroup)) < 0) { + break; + } + + int32_t tid = pCheckInfo->tableId.tid; + index = binarySearchForBlockImpl(pCheckInfo->pBlock, pCheckInfo->compIndex[tid].numOfSuperBlocks, pQueryHandle->order, key); + + if (type == QUERY_RANGE_GREATER_EQUAL) { + if (key <= pCheckInfo->pBlock[index].keyLast) { + break; + } else { + index = -1; + } + } else { + if (key >= pCheckInfo->pBlock[index].keyFirst) { + break; + } else { + index = -1; + } + } + } + + // failed to find qualified point in file, abort + if (index == -1) { + return false; + } + + assert(index >= 0 && index < pQueryHandle->numOfBlocks); + + // load first data block into memory failed, caused by disk block error + bool blockLoaded = false; + SArray *sa = NULL; + + // todo no need to loaded at all + cur->slot = index; + +// sa = getDefaultLoadColumns(pQueryHandle, true); + if (tsdbLoadDataBlock(&fileGroup->files[2], &pCheckInfo->pBlock[cur->slot], 1, fid, sa) == 0) { + blockLoaded = true; + } + + // dError("QInfo:%p fileId:%d total numOfBlks:%d blockId:%d load into memory failed due to error in disk files", + // GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx); + + // failed to load data from disk, abort current query + if (blockLoaded == false) { + return false; + } + + // todo search qualified points in blk, according to primary key (timestamp) column +// cur->pos = binarySearchForBlockImpl(ptsBuf->data, pBlocks->numOfPoints, key, pQueryHandle->order); + assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); + + filterDataInDataBlock(pQueryHandle, sa); + return pQueryHandle->realNumOfRows > 0; +} + +static bool hasMoreDataInFileForSingleTableModel(STsdbQueryHandle* pHandle) { + assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); + + STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); + SQueryFilePos* cur = &pHandle->cur; + + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + if (!pCheckInfo->checkFirstFileBlock) { + pCheckInfo->checkFirstFileBlock = true; + + if (pFileHandle != NULL) { + bool found = getQualifiedDataBlock(pHandle, pCheckInfo, 1); + if (found) { + return true; + } + } + + // no data in file, try cache + pHandle->cur.fid = -1; + return hasMoreDataInCacheForSingleModel(pHandle); + } else { // move to next data block in file or in cache + return moveToNextBlock(pHandle, 1); + } +} + +static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) { + size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo); + assert(numOfTables > 0); + + while(pHandle->activeIndex < numOfTables) { + STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + STable *pTable = pTableCheckInfo->pTableObj; + if (pTable->mem == NULL && pTable->imem == NULL) { + pHandle->activeIndex += 1; // try next table if exits + continue; + } + + // all data in mem are checked already. + if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { + pHandle->activeIndex += 1; // try next table if exits + continue; + } + + return true; + } + + // all tables has checked already + return false; +} + +// handle data in cache situation +bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; + if (pHandle->model == SINGLE_TABLE_MODEL) { + return hasMoreDataInFileForSingleTableModel(pHandle); + } else { + return hasMoreDataInCacheForMultiModel(pHandle); + } +} + static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pHandle) { int numOfRows = 0; @@ -360,9 +755,9 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max // copy data from cache into data block SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - STableIdInfo* idInfo = taosArrayGet(pHandle->pTableIdList, 0); - STable *pTable = pHandle->pTableQueryInfo->pTableObj; + STableCheckInfo* pTableQInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + STable *pTable = pTableQInfo->pTableObj; TSKEY skey = 0, ekey = 0; int32_t rows = 0; @@ -370,22 +765,22 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { if (pTable->mem != NULL) { // create mem table iterator if it is not created yet - if (pHandle->memIter == NULL) { - pHandle->memIter = tSkipListCreateIter(pTable->mem->pData); + if (pTableQInfo->iter == NULL) { + pTableQInfo->iter = tSkipListCreateIter(pTable->mem->pData); } - rows = tsdbReadRowsFromCache(pHandle->memIter, INT64_MAX, 2, &skey, &ekey, pHandle); + rows = tsdbReadRowsFromCache(pTableQInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle); } SDataBlockInfo blockInfo = { - .uid = idInfo->uid, - .sid = idInfo->sid, + .uid = pTable->tableId.uid, + .sid = pTable->tableId.tid, .size = rows, .window = {.skey = skey, .ekey = ekey} }; // update the last key value - pHandle->pTableQueryInfo->lastKey = ekey + 1; + pTableQInfo->lastKey = ekey + 1; return blockInfo; } @@ -412,6 +807,330 @@ SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr) {} -STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) {} +SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) {} + +static SArray* createTableIdArrayList(struct STsdbRepo* tsdb, int64_t uid) { + STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); + assert(pTable != NULL); //assert pTable is a super table + + size_t size = tSkipListGetSize(pTable->pIndex); + SArray* pList = taosArrayInit(size, sizeof(STableId)); + + SSkipListIterator* iter = tSkipListCreateIter(pTable->pIndex); + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + STable* t = *(STable**) SL_GET_NODE_DATA(pNode); + + taosArrayPush(pList, &t->tableId); + } + + return pList; +} + +typedef struct SSyntaxTreeFilterSupporter { + SSchema* pTagSchema; + int32_t numOfTags; + int32_t optr; +} SSyntaxTreeFilterSupporter; + +/** + * convert the result pointer to STabObj instead of tSkipListNode + * @param pRes + */ +static void tansformQueryResult(SArray* pRes) { + if (pRes == NULL || taosArrayGetSize(pRes) == 0) { + return; + } + + size_t size = taosArrayGetSize(pRes); + for (int32_t i = 0; i < size; ++i) { +// pRes->pRes[i] = ((tSkipListNode*)(pRes->pRes[i]))->pData; + } +} + +void tSQLListTraverseDestroyInfo(void* param) { + if (param == NULL) { + return; + } + + tQueryInfo* pInfo = (tQueryInfo*)param; + tVariantDestroy(&(pInfo->q)); + free(param); +} + +static char* convertTagQueryStr(const wchar_t* str, size_t len) { + char* mbs = NULL; + + if (len > 0) { + mbs = calloc(1, (len + 1) * TSDB_NCHAR_SIZE); + taosUcs4ToMbs((void*) str, len * TSDB_NCHAR_SIZE, mbs); //todo add log + } + + return mbs; +} + +static int32_t compareStrVal(const void* pLeft, const void* pRight) { + int32_t ret = strcmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +static int32_t compareWStrVal(const void* pLeft, const void* pRight) { + int32_t ret = wcscmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +static int32_t compareIntVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_INT64_VAL(pRight)); +} + +static int32_t compareIntDoubleVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_DOUBLE_VAL(pRight)); +} + +static int32_t compareDoubleVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_DOUBLE_VAL(pLeft), GET_DOUBLE_VAL(pRight)); +} + +static int32_t compareDoubleIntVal(const void* pLeft, const void* pRight) { + double ret = (*(double*)pLeft) - (*(int64_t*)pRight); + if (fabs(ret) < DBL_EPSILON) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const char* pattern = pRight; + const char* str = pLeft; + + int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const wchar_t* pattern = pRight; + const wchar_t* str = pLeft; + + int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { + __compar_fn_t comparator = NULL; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_BOOL: { + if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { + comparator = compareIntVal; + } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { + comparator = compareIntDoubleVal; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { + comparator = compareDoubleIntVal; + } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { + comparator = compareDoubleVal; + } + break; + } + + case TSDB_DATA_TYPE_BINARY: { + assert(filterType == TSDB_DATA_TYPE_BINARY); + + if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + comparator = compareStrPatternComp; + } else { /* normal relational comparator */ + comparator = compareStrVal; + } + + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + assert(filterType == TSDB_DATA_TYPE_NCHAR); + + if (optr == TSDB_RELATION_LIKE) { + comparator = compareWStrPatternComp; + } else { + comparator = compareWStrVal; + } + + break; + } + default: + comparator = compareIntVal; + break; + } + + return comparator; +} + +static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pSchema, int32_t* index, + int32_t* offset) { + *index = 0; + *offset = 0; + + // filter on table name(TBNAME) + if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { + *index = TSDB_TBNAME_COLUMN_INDEX; + *offset = TSDB_TBNAME_COLUMN_INDEX; + return; + } + + while ((*index) < pSupporter->numOfTags) { + if (pSupporter->pTagSchema[*index].bytes == pSchema->bytes && + pSupporter->pTagSchema[*index].type == pSchema->type && + strcmp(pSupporter->pTagSchema[*index].name, pSchema->name) == 0) { + break; + } else { + (*offset) += pSupporter->pTagSchema[(*index)++].bytes; + } + } +} + +void filterPrepare(void* expr, void* param) { + tSQLBinaryExpr *pExpr = (tSQLBinaryExpr*) expr; + if (pExpr->info != NULL) { + return; + } + + int32_t i = 0, offset = 0; + pExpr->info = calloc(1, sizeof(tQueryInfo)); + + tQueryInfo* pInfo = pExpr->info; + SSyntaxTreeFilterSupporter* pSupporter = (SSyntaxTreeFilterSupporter*)param; + + tVariant* pCond = pExpr->pRight->pVal; + SSchema* pSchema = pExpr->pLeft->pSchema; + + getTagColumnInfo(pSupporter, pSchema, &i, &offset); + assert((i >= 0 && i < TSDB_MAX_TAGS) || (i == TSDB_TBNAME_COLUMN_INDEX)); + assert((offset >= 0 && offset < TSDB_MAX_TAGS_LEN) || (offset == TSDB_TBNAME_COLUMN_INDEX)); + + pInfo->sch = *pSchema; + pInfo->colIdx = i; + pInfo->optr = pExpr->nSQLBinaryOptr; + pInfo->offset = offset; + pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr); + + tVariantAssign(&pInfo->q, pCond); + tVariantTypeSetType(&pInfo->q, pInfo->sch.type); +} + +bool tSkipListNodeFilterCallback(const void* pNode, void* param) { + tQueryInfo* pInfo = (tQueryInfo*)param; + + STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); + + char* val = dataRowTuple(pTable->tagVal); // todo not only the first column + int8_t type = pInfo->sch.type; + + int32_t ret = 0; + if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) { + ret = pInfo->compare(val, pInfo->q.pz); + } else { + tVariant t = {0}; + tVariantCreateFromBinary(&t, val, (uint32_t) pInfo->sch.bytes, type); + + ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); + } + + switch (pInfo->optr) { + case TSDB_RELATION_EQUAL: { + return ret == 0; + } + case TSDB_RELATION_NOT_EQUAL: { + return ret != 0; + } + case TSDB_RELATION_LARGE_EQUAL: { + return ret >= 0; + } + case TSDB_RELATION_LARGE: { + return ret > 0; + } + case TSDB_RELATION_LESS_EQUAL: { + return ret <= 0; + } + case TSDB_RELATION_LESS: { + return ret < 0; + } + case TSDB_RELATION_LIKE: { + return ret == 0; + } + + default: + assert(false); + } + return true; +} + +static int32_t doQueryTableList(STable* pSTable, SArray* pRes, const char* pCond) { + STColumn* stcol = schemaColAt(pSTable->tagSchema, 0); + + tSQLBinaryExpr* pExpr = NULL; + tSQLBinaryExprFromString(&pExpr, stcol, schemaNCols(pSTable->tagSchema), pCond, strlen(pCond)); + + // failed to build expression, no result, return immediately + if (pExpr == NULL) { + mError("table:%" PRIu64 ", no result returned, error in super table query expression:%s", pSTable->tableId.uid, pCond); + tfree(pCond); + + return TSDB_CODE_OPS_NOT_SUPPORT; + } + + // query according to the binary expression + SSyntaxTreeFilterSupporter s = {.pTagSchema = stcol, .numOfTags = schemaNCols(pSTable->tagSchema)}; + + SBinaryFilterSupp supp = { + .fp = (__result_filter_fn_t)tSkipListNodeFilterCallback, + .setupInfoFn = (__do_filter_suppl_fn_t)filterPrepare, + .pExtInfo = &s + }; + + tSQLBinaryExprTraverse(pExpr, pSTable->pIndex, pRes, &supp); + tSQLBinaryExprDestroy(&pExpr, tSQLListTraverseDestroyInfo); + + tansformQueryResult(pRes); + + return TSDB_CODE_SUCCESS; +} -STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond) {} +SArray *tsdbQueryTableList(struct STsdbRepo* tsdb, int64_t uid, const wchar_t *pTagCond, size_t len) { + // no condition, all tables created according to the stable will involved in querying + if (pTagCond == NULL || wcslen(pTagCond) == 0) { + return createTableIdArrayList(tsdb, uid); + } else { + char* str = convertTagQueryStr(pTagCond, len); + SArray* result = taosArrayInit(8, POINTER_BYTES); + + STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); + assert(pSTable != NULL); + + if (doQueryTableList(pSTable, result, str) == TSDB_CODE_SUCCESS) { + return result; + } + } +} diff --git a/src/vnode/tsdb/tests/CMakeLists.txt b/src/vnode/tsdb/tests/CMakeLists.txt index 51c15bce203eb920d7f377d261322047ccd008dc..ee1aaba8cd8c623b29f2d1200c640da2a62dfe6e 100644 --- a/src/vnode/tsdb/tests/CMakeLists.txt +++ b/src/vnode/tsdb/tests/CMakeLists.txt @@ -3,9 +3,4 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) add_executable(tsdbTests ${SOURCE_LIST}) target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb) -add_test( - NAME - unit - COMMAND - ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests -) \ No newline at end of file +add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests) \ No newline at end of file diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index bc6532984fddd74934c9f87a9e5f1086f08faef3..6cfe0e626dd217b70597c2f75f2f7a575472d1e3 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -49,6 +49,7 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) { ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0); } +// TEST(TsdbTest, DISABLED_createRepo) { TEST(TsdbTest, createRepo) { STsdbCfg config; @@ -87,6 +88,7 @@ TEST(TsdbTest, createRepo) { double stime = getCurTime(); for (int k = 0; k < nRows/rowsPerSubmit; k++) { + memset((void *)pMsg, 0, sizeof(SSubmitMsg)); SSubmitBlk *pBlock = pMsg->blocks; pBlock->uid = 987607499877672L; pBlock->tid = 0; @@ -108,6 +110,9 @@ TEST(TsdbTest, createRepo) { } pBlock->len += dataRowLen(row); } + pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; + pMsg->numOfBlocks = 1; + pBlock->len = htonl(pBlock->len); pBlock->numOfRows = htonl(pBlock->numOfRows); pBlock->uid = htobe64(pBlock->uid); @@ -116,7 +121,6 @@ TEST(TsdbTest, createRepo) { pBlock->sversion = htonl(pBlock->sversion); pBlock->padding = htonl(pBlock->padding); - pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; pMsg->length = htonl(pMsg->length); pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); pMsg->compressed = htonl(pMsg->numOfBlocks); @@ -126,15 +130,17 @@ TEST(TsdbTest, createRepo) { double etime = getCurTime(); - printf("Spent %f seconds to write %d records\n", etime - stime, nRows); - + void *ptr = malloc(150000); + free(ptr); + printf("Spent %f seconds to write %d records\n", etime - stime, nRows); - // tsdbTriggerCommit(pRepo); + tsdbCloseRepo(pRepo); } -TEST(TsdbTest, DISABLED_openRepo) { +// TEST(TsdbTest, DISABLED_openRepo) { +TEST(TsdbTest, openRepo) { tsdb_repo_t *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0"); ASSERT_NE(pRepo, nullptr); } diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 6fcedb8123cb2ea8e6bb6b89a3b7d45c96bcd3f6..b621781a3c81e4155841ac5d4ec31aae6a38518e 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -46,7 +46,8 @@ int main(int argc, char *argv[]) { } printf("success to connect to server\n"); - int32_t code = taos_query(taos, "select * from test.t1"); +// int32_t code = taos_query(taos, "insert into test.tm2 values(now, 1)(now+1m,2)(now+2m,3) (now+3m, 4) (now+4m, 5);"); + int32_t code = taos_query(taos, "insert into test.tm2 values(now, 99)"); if (code != 0) { printf("failed to execute query, reason:%s\n", taos_errstr(taos)); }