From 5f0a05163d6280dcf151afa1c0cf9e3c17e4fc6f Mon Sep 17 00:00:00 2001 From: hjLiao Date: Wed, 13 May 2020 11:52:33 +0800 Subject: [PATCH] [td-225] fix bugs for leastsquares query. --- src/client/inc/tscUtil.h | 2 +- src/client/src/tscFunctionImpl.c | 6 +++- src/client/src/tscSQLParser.c | 2 ++ src/client/src/tscUtil.c | 27 ++++++++--------- src/common/inc/tname.h | 1 + src/inc/taosdef.h | 5 ++-- src/query/src/queryExecutor.c | 31 ++++++++++---------- src/tsdb/src/tsdbRead.c | 5 ++-- tests/script/general/compute/leastsquare.sim | 12 ++++---- 9 files changed, 50 insertions(+), 41 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 71c24501d1..2a37f3fc7c 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -183,7 +183,7 @@ void tscSqlExprInfoDestroy(SArray* pExprInfo); SColumn* tscColumnClone(const SColumn* src); SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); -void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex); +SArray* tscColumnListClone(const SArray* src, int16_t tableIndex); void tscColumnListDestroy(SArray* pColList); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 6fb8df2444..5675416e6b 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2904,7 +2904,11 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { param[1][2] /= param[1][1]; - sprintf(pCtx->aOutputBuf, "(%lf, %lf)", param[0][2], param[1][2]); + int32_t maxOutputSize = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE; + size_t n = snprintf(varDataVal(pCtx->aOutputBuf), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", + param[0][2], param[1][2]); + + varDataSetLen(pCtx->aOutputBuf, n); doFinalizer(pCtx); } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3dbd5d36bd..86a74f0115 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3792,6 +3792,8 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSQLExprDestroy(p1); tExprTreeDestroy(&p, NULL); + + taosArrayDestroy(colList); } pCondExpr->pTagCond = NULL; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index bea61f12c4..171d40b932 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1209,18 +1209,18 @@ void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex) { } } -void tscColumnListDestroy(SArray* pColumnBaseInfo) { - if (pColumnBaseInfo == NULL) { +void tscColumnListDestroy(SArray* pColumnList) { + if (pColumnList == NULL) { return; } - size_t num = taosArrayGetSize(pColumnBaseInfo); + size_t num = taosArrayGetSize(pColumnList); for (int32_t i = 0; i < num; ++i) { - SColumn* pCol = taosArrayGetP(pColumnBaseInfo, i); + SColumn* pCol = taosArrayGetP(pColumnList, i); tscColumnDestroy(pCol); } - taosArrayDestroy(pColumnBaseInfo); + taosArrayDestroy(pColumnList); } /* @@ -1572,7 +1572,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { assert(pQueryInfo->exprList == NULL); pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { @@ -1666,8 +1666,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST assert(pTableMetaInfo != NULL); if (name != NULL) { - assert(strlen(name) <= TSDB_TABLE_ID_LEN); - strcpy(pTableMetaInfo->name, name); + strncpy(pTableMetaInfo->name, name, TSDB_TABLE_ID_LEN); } pTableMetaInfo->pTableMeta = pTableMeta; @@ -1678,10 +1677,9 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST memcpy(pTableMetaInfo->vgroupList, vgroupList, size); } - if (pTagCols == NULL) { - pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); - } else { - pTableMetaInfo->tagColList = taosArrayClone(pTagCols); + pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); + if (pTagCols != NULL) { + tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } pQueryInfo->numOfTables += 1; @@ -1794,7 +1792,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void tscFreeSqlObj(pNew); return NULL; } - + tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); // set the correct query type @@ -1853,7 +1851,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (pPrevSql == NULL) { STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name); - + // todo handle error + assert(pTableMeta != NULL); pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index a629128324..810d7f492c 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -23,4 +23,5 @@ void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); + #endif // TDENGINE_NAME_H diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 33a0e4f2c6..23436fe6a5 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -32,6 +32,9 @@ extern "C" { #define TSKEY int64_t #endif +#define TSWINDOW_INITIALIZER {INT64_MIN, INT64_MAX}; +#define TSKEY_INITIAL_VAL INT64_MIN + // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR typedef int32_t VarDataOffsetT; typedef int16_t VarDataLenT; @@ -341,8 +344,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_DBS 100 #define TSDB_MAX_VGROUPS 1000 #define TSDB_MAX_SUPER_TABLES 100 -#define TSDB_MAX_NORMAL_TABLES 1000 -#define TSDB_MAX_CHILD_TABLES 100000 #define TSDB_PORT_DNODESHELL 0 #define TSDB_PORT_DNODEDNODE 5 diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index fb19d0f50c..4b48d79316 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -507,7 +507,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t w.ekey = pQuery->window.ekey; } - assert(ts >= w.skey && ts <= w.ekey && w.skey != 0); + assert(ts >= w.skey && ts <= w.ekey); return w; } @@ -624,7 +624,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); } else { // set the current index to be the last unclosed window int32_t i = 0; - int64_t skey = 0; + int64_t skey = TSKEY_INITIAL_VAL; for (i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; @@ -642,7 +642,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, } // all windows are closed, set the last one to be the skey - if (skey == 0) { + if (skey == TSKEY_INITIAL_VAL) { assert(i == pWindowResInfo->size); pWindowResInfo->curIndex = pWindowResInfo->size - 1; } else { @@ -660,7 +660,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, qTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, n); } - assert(pWindowResInfo->prevSKey != 0); + assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); } static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, @@ -3080,6 +3080,9 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { int32_t functId = pQuery->pSelectExpr[j].base.functionId; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; + if (pCtx->resultInfo == NULL) { + continue; // resultInfo is NULL, means no data checked in previous scan + } if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { @@ -3590,7 +3593,6 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { if (pTableQueryInfo->queryRangeSet) { pTableQueryInfo->lastKey = key; } else { -// pQuery->window.skey = key; pTableQueryInfo->win.skey = key; STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey}; @@ -3613,18 +3615,16 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp - if (pWindowResInfo->prevSKey == 0) { - if (QUERY_IS_ASC_QUERY(pQuery)) { - pWindowResInfo->prevSKey = w.skey; - } else { + if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { assert(win.ekey == pQuery->window.skey); - pWindowResInfo->prevSKey = w.skey; } + + pWindowResInfo->prevSKey = w.skey; } pTableQueryInfo->queryRangeSet = 1; pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - pTableQueryInfo->win.skey = pTableQueryInfo->win.skey; } } @@ -4057,10 +4057,11 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) { * pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is * not valid. otherwise, we only forward pQuery->limit.offset number of points */ - assert(pRuntimeEnv->windowResInfo.prevSKey == 0); + assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); - TSKEY skey1, ekey1; - STimeWindow w = {0}; + TSKEY skey1, ekey1; + STimeWindow w = TSWINDOW_INITIALIZER; + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -4730,7 +4731,7 @@ static void doRestoreContext(SQInfo *pQInfo) { SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); if (pRuntimeEnv->pTSBuf != NULL) { - pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; + SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); } switchCtxOrder(pRuntimeEnv); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 84615ff8c2..6a5a4b01bb 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1346,7 +1346,7 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { int32_t type = 0; int32_t bytes = 0; - if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor extract method , to queryExecutor to generate tags values f1 = (char*) pTable1->name; f2 = (char*) pTable2->name; type = TSDB_DATA_TYPE_BINARY; @@ -1354,7 +1354,8 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { } else { STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex); bytes = pCol->bytes; - + type = pCol->type; + f1 = tdGetRowDataOfCol(pTable1->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); f2 = tdGetRowDataOfCol(pTable2->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); } diff --git a/tests/script/general/compute/leastsquare.sim b/tests/script/general/compute/leastsquare.sim index bbadd849cb..5a28e74bff 100644 --- a/tests/script/general/compute/leastsquare.sim +++ b/tests/script/general/compute/leastsquare.sim @@ -48,41 +48,41 @@ $tb = $tbPrefix . $i sql select leastsquares(tbcol, 1, 1) from $tb print ===> $data00 -if $data00 != @(1.000000, 1.000000)@ then +if $data00 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step3 sql select leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m print ===> $data00 -if $data00 != @(1.000000, 1.000000)@ then +if $data00 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step4 sql select leastsquares(tbcol, 1, 1) as b from $tb print ===> $data00 -if $data00 != @(1.000000, 1.000000)@ then +if $data00 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step5 sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1m) print ===> $data01 -if $data01 != @(1.000000, 1.000000)@ then +if $data01 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1d) print ===> $data01 -if $data01 != @(1.000000, 1.000000)@ then +if $data01 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step6 sql select leastsquares(tbcol, 1, 1) as b from $tb where ts < now + 4m interval(1m) print ===> $data01 -if $data01 != @(1.000000, 1.000000)@ then +if $data01 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print ===> $rows -- GitLab