diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 849d31f93e9eaa0f9c834034ee277392201df0cf..5c6ffd4b10176d4745e825861994af12dd704576 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -76,15 +76,19 @@ IF ("${CPUTYPE}" STREQUAL "") IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") MESSAGE(STATUS "The current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") + SET(TD_INTEL_64 TRUE) ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "The current platform is x86") SET(PLATFORM_ARCH_STR "i386") + SET(TD_INTEL_32 TRUE) ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "The current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") + SET(TD_ARM_32 TRUE) ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") MESSAGE(STATUS "The current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") + SET(TD_ARM_64 TRUE) ENDIF () ELSE () # if generate ARM version: @@ -92,18 +96,23 @@ ELSE () IF (${CPUTYPE} MATCHES "aarch32") SET(PLATFORM_ARCH_STR "arm") MESSAGE(STATUS "input cpuType: aarch32") + SET(TD_ARM_32 TRUE) ELSEIF (${CPUTYPE} MATCHES "aarch64") SET(PLATFORM_ARCH_STR "arm64") MESSAGE(STATUS "input cpuType: aarch64") + SET(TD_ARM_64 TRUE) ELSEIF (${CPUTYPE} MATCHES "mips64") SET(PLATFORM_ARCH_STR "mips") MESSAGE(STATUS "input cpuType: mips64") + SET(TD_MIPS_64 TRUE) ELSEIF (${CPUTYPE} MATCHES "x64") SET(PLATFORM_ARCH_STR "amd64") MESSAGE(STATUS "input cpuType: x64") + SET(TD_INTEL_64 TRUE) ELSEIF (${CPUTYPE} MATCHES "x86") SET(PLATFORM_ARCH_STR "i386") MESSAGE(STATUS "input cpuType: x86") + SET(TD_INTEL_32 TRUE) ELSE () MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) ENDIF () diff --git a/docs/zh/12-taos-sql/12-interval.md b/docs/zh/12-taos-sql/12-interval.md index 0fece8a7d44bbc92f2bcd6eb3dea40f1cc305016..8bf0c578e79009ba9ca6999ffb27a05b223d25b5 100644 --- a/docs/zh/12-taos-sql/12-interval.md +++ b/docs/zh/12-taos-sql/12-interval.md @@ -3,9 +3,9 @@ sidebar_label: 时序数据特色查询 title: 时序数据特色查询 --- -TDengine是专为时序数据而研发的大数据平台，存储和计算都针对时序数据的特定进行了量身定制，在支持标准SQL的基础之上，还提供了一系列贴合时序业务场景的特色查询语法，极大的方便时序场景的应用开发。 +TDengine 是专为时序数据而研发的大数据平台，存储和计算都针对时序数据的特定进行了量身定制，在支持标准 SQL 的基础之上，还提供了一系列贴合时序业务场景的特色查询语法，极大的方便时序场景的应用开发。 -TDengine提供的特色查询包括标签切分查询和窗口切分查询。 +TDengine 提供的特色查询包括标签切分查询和窗口切分查询。 ## 标签切分查询 @@ -14,13 +14,14 @@ TDengine提供的特色查询包括标签切分查询和窗口切分查询。 ```sql PARTITION BY tag_list ``` -其中 `tag_list` 是标签列的列表，还可以包括tbname伪列。 -TDengine按如下方式处理标签切分子句： +其中 `tag_list` 是标签列的列表，还可以包括 tbname 伪列。 + +TDengine 按如下方式处理标签切分子句： 标签切分子句位于 `WHERE` 子句之后，且不能和 `JOIN` 子句一起使用。 标签切分子句将超级表数据按指定的标签组合进行切分，然后对每个切分的分片进行指定的计算。计算由之后的子句定义（窗口子句、`GROUP BY` 子句或`SELECT` 子句）。 -标签切分子句可以和窗口切分子句（或 `GROUP BY` 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，下面的示例将数据按标签 `location` 进行分组，并对每个组按10分钟进行降采样，取其最大值。 +标签切分子句可以和窗口切分子句（或 `GROUP BY` 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，下面的示例将数据按标签 `location` 进行分组，并对每个组按 10 分钟进行降采样，取其最大值。 ```sql select max(current) from meters partition by location interval(10m) @@ -42,27 +43,31 @@ SELECT function_list FROM tb_name 在上述语法中的具体限制如下 ### 窗口切分查询中使用函数的限制 + - 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：DIFF 以及四则运算）。 - 此外 LAST_ROW 查询也不能与窗口聚合同时出现。 - 标量函数（如：CEIL/FLOOR 等）也不能使用在窗口聚合查询中。 ### 窗口子句的规则 -- 窗口子句位于标签切分子句之后，GROUP BY子句之前，且不可以和GROUP BY子句一起使用。 -- 窗口子句将数据按窗口进行切分，对每个窗口进行SELECT列表中的表达式的计算，SELECT列表中的表达式只能包含： + +- 窗口子句位于标签切分子句之后，GROUP BY 子句之前，且不可以和 GROUP BY 子句一起使用。 +- 窗口子句将数据按窗口进行切分，对每个窗口进行 SELECT 列表中的表达式的计算，SELECT 列表中的表达式只能包含： - 常量。 - 聚集函数。 - 包含上面表达式的表达式。 -- 窗口子句不可以和GROUP BY子句一起使用。 +- 窗口子句不可以和 GROUP BY 子句一起使用。 - WHERE 语句可以指定查询的起止时间和其他过滤条件。 ### FILL 子句 - FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： - 1. 不进行填充：NONE（默认填充模式）。 - 2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如FILL(VALUE, 1.23)，相应列为INT类型，则填充值为1。 - 3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 - 4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 - 5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 - 6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 + +FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： + +1. 不进行填充：NONE（默认填充模式）。 +2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1。 +3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 +4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 +5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 +6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 :::info @@ -93,6 +98,7 @@ SELECT COUNT(*) FROM temp_tb_1 INTERVAL(1m) SLIDING(2m); ``` 使用时间窗口需要注意： + - 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。 - 使用 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。 - 返回的结果中时间序列严格单调递增。 diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/docs/zh/12-taos-sql/13-operators.md b/docs/zh/12-taos-sql/16-operators.md similarity index 100% rename from docs/zh/12-taos-sql/13-operators.md rename to docs/zh/12-taos-sql/16-operators.md diff --git a/docs/zh/12-taos-sql/16-json.md b/docs/zh/12-taos-sql/17-json.md similarity index 100% rename from docs/zh/12-taos-sql/16-json.md rename to docs/zh/12-taos-sql/17-json.md diff --git a/docs/zh/12-taos-sql/14-limit.md b/docs/zh/12-taos-sql/19-limit.md similarity index 100% rename from docs/zh/12-taos-sql/14-limit.md rename to docs/zh/12-taos-sql/19-limit.md diff --git a/docs/zh/12-taos-sql/22-information.md b/docs/zh/12-taos-sql/22-information.md new file mode 100644 index 0000000000000000000000000000000000000000..0695aa5172f0632cdfb0b388791f33a554f86d36 --- /dev/null +++ b/docs/zh/12-taos-sql/22-information.md @@ -0,0 +1,5 @@ +--- +sidebar_label: Information内置数据库 +title: Information内置数据库 +--- + diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5aee218ddc14b6ce5ef92d47321a46457f03c53d..930b7be3efd5d31b6849668664b6f277a8133f19 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -80,8 +80,6 @@ typedef struct { SArray* pTableList; SHashObj* map; // speedup acquire the tableQueryInfo by table uid bool needSortTableByGroupId; - void* pTagCond; - void* pTagIndexCond; uint64_t suid; } STableListInfo; @@ -268,26 +266,6 @@ typedef struct SSortExecInfo { int32_t readBytes; // read io bytes } SSortExecInfo; -//====================================================================================================================== -// for grant -typedef enum { - TSDB_GRANT_ALL, - TSDB_GRANT_TIME, - TSDB_GRANT_USER, - TSDB_GRANT_DB, - TSDB_GRANT_TIMESERIES, - TSDB_GRANT_DNODE, - TSDB_GRANT_ACCT, - TSDB_GRANT_STORAGE, - TSDB_GRANT_SPEED, - TSDB_GRANT_QUERY_TIME, - TSDB_GRANT_CONNS, - TSDB_GRANT_STREAMS, - TSDB_GRANT_CPU_CORES, -} EGrantType; - -int32_t grantCheck(EGrantType grant); - #ifdef __cplusplus } #endif diff --git a/include/common/tgrant.h b/include/common/tgrant.h new file mode 100644 index 0000000000000000000000000000000000000000..ad23c661b1e76b12128a2946db76f2ce44ed7663 --- /dev/null +++ b/include/common/tgrant.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_GRANT_H_ +#define _TD_COMMON_GRANT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +typedef enum { + TSDB_GRANT_ALL, + TSDB_GRANT_TIME, + TSDB_GRANT_USER, + TSDB_GRANT_DB, + TSDB_GRANT_TIMESERIES, + TSDB_GRANT_DNODE, + TSDB_GRANT_ACCT, + TSDB_GRANT_STORAGE, + TSDB_GRANT_SPEED, + TSDB_GRANT_QUERY_TIME, + TSDB_GRANT_CONNS, + TSDB_GRANT_STREAMS, + TSDB_GRANT_CPU_CORES, +} EGrantType; + +int32_t grantCheck(EGrantType grant); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_GRANT_H_*/ \ No newline at end of file diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d195bf5407e556d2f162d43a188c9761e7a95758..2f4c80f02551c696bc66d725eaadbd68de86463a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2071,9 +2071,10 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp); // TDMT_VND_ALTER_TABLE ===================== typedef struct { - char* tbName; - int8_t action; - char* colName; + char* tbName; + int8_t action; + char* colName; + int32_t colId; // TSDB_ALTER_TABLE_ADD_COLUMN int8_t type; int8_t flags; diff --git a/include/common/ttime.h b/include/common/ttime.h index de55b016cdacf2acb43c9f4c0b26599a310eb619..2f4129f97985b130634093520cc3598609fa93c7 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -63,12 +63,13 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : 1000000000; time_t t = taosTime(NULL); - struct tm* tm = taosLocalTime(&t, NULL); - tm->tm_hour = 0; - tm->tm_min = 0; - tm->tm_sec = 0; + struct tm tm; + taosLocalTime(&t, &tm); + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; - return (int64_t)taosMktime(tm) * factor; + return (int64_t)taosMktime(&tm) * factor; } int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 3bd02b1b48825190a9c893b9b38896bd2d70b594..1c3e5903f65369ca9ea3d6cc40f0b5ea3fb9d486 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -26,6 +26,12 @@ extern "C" { #define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN +typedef enum EDataOrderLevel { + DATA_ORDER_LEVEL_NONE = 1, + DATA_ORDER_LEVEL_IN_BLOCK, + DATA_ORDER_LEVEL_IN_GROUP +} EDataOrderLevel; + typedef struct SLogicNode { ENodeType type; SNodeList* pTargets; // SColumnNode @@ -36,6 +42,8 @@ typedef struct SLogicNode { uint8_t precision; SNode* pLimit; SNode* pSlimit; + EDataOrderLevel requireDataOrder; // requirements for input data + EDataOrderLevel resultDataOrder; // properties of the output data } SLogicNode; typedef enum EScanType { @@ -78,7 +86,7 @@ typedef struct SScanLogicNode { SNodeList* pGroupTags; bool groupSort; int8_t cacheLastMode; - bool hasNormalCols; // neither tag column nor primary key tag column + bool hasNormalCols; // neither tag column nor primary key tag column } SScanLogicNode; typedef struct SJoinLogicNode { @@ -317,6 +325,7 @@ typedef STableScanPhysiNode SStreamScanPhysiNode; typedef struct SProjectPhysiNode { SPhysiNode node; SNodeList* pProjections; + bool mergeDataBlock; } SProjectPhysiNode; typedef struct SIndefRowsFuncPhysiNode { diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index ab53c37279317b0266b306ffe745b9dd299bff7d..770b0442ea5413399b543545369246cf5346875b 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -116,6 +116,13 @@ int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 5b845fd455622cf5046a5949cb2845f787850988..e73888d9ba020f7c1f57fe0ceda863861337dfba 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -363,7 +363,11 @@ tmq_list_t* tmq_list_new() { int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; - char* topic = strdup(src); + if (src == NULL || src[0] == 0) return -1; + char* topic = strdup(src); + if (topic[0] != '`') { + strtolower(topic, src); + } if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d9ac84ec06054922552ab17304ba8262874905c5..c674728fe60ebe64fe310fe16f84574290277aa2 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -228,7 +228,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui uint32_t finalNumOfRows = numOfRow1 + numOfRow2; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - if (finalNumOfRows > *capacity || numOfRow1 == 0) { + if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) { char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -262,7 +262,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); pColumnInfoData->varmeta.length = len + oldLen; } else { - if (finalNumOfRows > *capacity || numOfRow1 == 0) { + if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) { ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { @@ -1661,9 +1661,6 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } */ -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -1674,9 +1671,9 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { ms += 1000; } } - - struct tm* ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm ptm = {0}; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index d728bbe49e9aecda349c34e5d9ec8528172ec271..944ee6a7318966a66d12c0677e8851672501dfa0 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -902,7 +902,7 @@ const char* fmtts(int64_t ts) { void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) { char ts[40] = {0}; - struct tm* ptm; + struct tm ptm; int32_t fractionLen; char* format = NULL; @@ -939,10 +939,10 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) assert(false); } - ptm = taosLocalTime(", NULL); - int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm); + taosLocalTime(", &ptm); + int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); length += snprintf(ts + length, fractionLen, format, mod); - length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm); + length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); tstrncpy(buf, ts, bufLen); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 06c782e2178ac29e87de8500ce2b22235824fe7e..5ad13e383a648b7703a7a247d182242950008750 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -153,9 +153,15 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp switch (qtype) { case QUERY_QUEUE: - vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); - dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pQueryQ, pMsg); + if ((pMsg->msgType == TDMT_SCH_QUERY) && (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS)) { + terrno = TSDB_CODE_GRANT_EXPIRED; + code = terrno; + dDebug("vgId:%d, msg:%p put into vnode-query queue failed since %s", pVnode->vgId, pMsg, terrstr()); + } else { + vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); + dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pQueryQ, pMsg); + } break; case STREAM_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg); @@ -166,9 +172,24 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - - dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pWriteQ, pMsg); + if ((pMsg->msgType == TDMT_VND_SUBMIT) && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) { + terrno = TSDB_CODE_VND_NO_WRITE_AUTH; + code = terrno; + dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr()); + } else { + dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pWriteQ, pMsg); +#if 0 // tests for batch writes + if (pMsg->msgType == TDMT_VND_CREATE_TABLE) { + SRpcMsg *pDup = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + memcpy(pDup, pMsg, sizeof(SRpcMsg)); + pDup->pCont = rpcMallocCont(pMsg->contLen); + memcpy(pDup->pCont, pMsg->pCont, pMsg->contLen); + pDup->info.handle = NULL; + taosWriteQitem(pVnode->pWriteQ, pDup); + } +#endif + } break; case SYNC_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8ad1ac56e4ceb03ee47db2d9c8d2f2313b339500..f72b69a7de243d27d202a08fbb21561a3e1b4224 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -24,6 +24,7 @@ #include "tcache.h" #include "tdatablock.h" #include "tglobal.h" +#include "tgrant.h" #include "tqueue.h" #include "ttime.h" #include "version.h" diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e3b0f3c157ab15823792eb52af50acd671194f5a..b4b1b383e7e0f06fc9c89ce46e10493947502348 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -509,11 +509,10 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { SUserObj *pUser = NULL; SCreateDbReq createReq = {0}; - // code = grantCheck(TSDB_GRANT_DB); - // if (code != 0) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_DB)) != 0) { + code = terrno; + goto _OVER; + } if (tDeserializeSCreateDbReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d4c043028d850bc900ef3415dc2a55a226246ec7..f26c1a7d3224d48579a19af5c1f85ffcc525e56a 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -621,11 +621,10 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SCreateDnodeReq createReq = {0}; - // code = grantCheck(TSDB_GRANT_DNODE); - // if (code != TSDB_CODE_SUCCESS) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0) { + code = terrno; + goto _OVER; + } if (tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index d67e4e8783f107a2dc2333e32151380456f22b81..05e197150eb3fe099574d1f15a868a92305b007a 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -298,7 +298,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, + pConsumerEp->consumerId); } imbCnt++; } @@ -312,7 +313,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, + pConsumerEp->consumerId); } } } @@ -354,7 +356,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); } } @@ -371,8 +374,14 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ASSERT(pConsumerEp->consumerId > 0); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; + if (pRebVg->newConsumerId == pRebVg->oldConsumerId) { + mInfo("mq rebalance: skip vg %d for same consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); + continue; + } taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); } } else { // if all consumer is removed, put all vg into unassigned diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5c361ed28d51252915214aa1bba25135880e2b89..0452659d47ab333be6de7da5350787b7c09102a9 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -363,11 +363,10 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - // code = grantCheck(TSDB_GRANT_USER); - // if (code != TSDB_CODE_SUCCESS) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_USER)) != 0) { + code = terrno; + goto _OVER; + } code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 57141bbbb52001a05a9af11307b883d761918e37..97eff4804d5c557c7a984dcd1bec4dcd9b1edc7b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -27,6 +27,7 @@ #include "wal.h" #include "tcommon.h" +#include "tgrant.h" #include "tfs.h" #include "tmsg.h" #include "trow.h" diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 1c8434594d7928d5e150234f4c7e6ef4e03f9fc9..4673b01d27b58602a9ccfdd19412fe25f5c67003 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -43,9 +43,9 @@ typedef struct SRSmaInfo SRSmaInfo; typedef struct SRSmaInfoItem SRSmaInfoItem; struct SSmaEnv { - SRWLatch lock; - int8_t type; - SSmaStat *pStat; + SRWLatch lock; + int8_t type; + SSmaStat *pStat; }; typedef struct { @@ -103,10 +103,15 @@ struct SRSmaInfoItem { }; struct SRSmaInfo { - STSchema *pTSchema; - int64_t suid; + STSchema *pTSchema; + int64_t suid; + int8_t delFlag; + T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; }; +#define RSMA_INFO_HEAD_LEN 24 +#define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) +#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) enum { TASK_TRIGGER_STAT_INIT = 0, @@ -120,8 +125,8 @@ enum { enum { RSMA_ROLE_CREATE = 0, RSMA_ROLE_DROP = 1, - RSMA_ROLE_FETCH = 2, - RSMA_ROLE_SUBMIT = 3, + RSMA_ROLE_SUBMIT = 2, + RSMA_ROLE_FETCH = 3, RSMA_ROLE_ITERATE = 4, }; @@ -134,6 +139,8 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg); int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat); int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); +int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo); +int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo); void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); @@ -193,6 +200,7 @@ void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t le static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); +void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); @@ -258,8 +266,9 @@ void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); void tdCloseTFile(STFile *pTFile); void tdDestroyTFile(STFile *pTFile); -void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName); -void tdGetVndDirName(int32_t vgId,const char *pdname, const char *dname, bool endWithSep, char *outputName); +void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, + char *outputName); +void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a7fef795ee4d43b6e47e33b675568af7a44d50b7..776a3c5b7f41b501a1010aa3499d02a64d296d04 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -205,7 +205,7 @@ _query: } tDecoderInit(&dc, pData, nData); - tDecodeSSchemaWrapper(&dc, &schema); + tDecodeSSchemaWrapperEx(&dc, &schema); pSchema = tCloneSSchemaWrapper(&schema); tDecoderClear(&dc); @@ -470,9 +470,9 @@ int64_t metaGetTbNum(SMeta *pMeta) { } // N.B. Called by statusReq per second -int64_t metaGetTimeSeriesNum(SMeta *pMeta) { +int64_t metaGetTimeSeriesNum(SMeta *pMeta) { // TODO - return 400; + return 400; } typedef struct { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 29c10c40c2b00d8e9006fc4b9fe2505059c3e8a6..71345df747eaeb3a7df77de63f9db748ff69f0e4 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -277,7 +277,7 @@ _drop_super_table: _exit: tdbFree(pKey); tdbFree(pData); - metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; } diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index cdaaf2bbdb327d0940f7b44735182e9474e74a29..2174a479e7e2f998941b4dc87f8ce66b4c70603f 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -191,7 +191,7 @@ static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) { if ((pDir = taosOpenDir(dir)) == NULL) { regfree(®ex); terrno = TAOS_SYSTEM_ERROR(errno); - smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); + smaDebug("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); return TSDB_CODE_FAILED; } @@ -392,5 +392,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { // step 2: cleanup outdated qtaskinfo files tdCleanupQTaskInfoFiles(pSma, pRSmaStat); + atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 577d5fd4fabccee60aa68c9c21bc409c9dbffbd7..23706d54e0df5947546da63399375c3b5b5f03bf 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -169,6 +169,26 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { return 0; } +int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { + if (!pRSmaInfo) return 0; + + int ref = T_REF_INC(pRSmaInfo); + smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); + return 0; +} + +int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { + if (!pRSmaInfo) return 0; + + int ref = T_REF_DEC(pRSmaInfo); + smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); + + if (ref == 0) { + tdRemoveRSmaInfoBySuid(pSma, pRSmaInfo->suid); + } + return 0; +} + static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) { ASSERT(pSmaStat != NULL); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 32d6dee57f24642ef42807c29086ec6602fcec3c..5b1a87be203c30dbc1638eb58a17e0777364f302 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -24,6 +24,7 @@ SSmaMgmt smaMgmt = { }; #define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver" +#define TD_RSMAINFO_DEL_FILE "rsmainfo.del" typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; @@ -48,14 +49,11 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed); static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed); - - static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) { // adapt accordingly if definition of SRSmaInfo update SRSmaInfo *pResult = NULL; - int32_t rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *); ASSERT(pItem->level == TSDB_RETENTION_L1 || pItem->level == TSDB_RETENTION_L2); - pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + rsmaInfoHeadLen)); + pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + RSMA_INFO_HEAD_LEN)); ASSERT(pResult->pTSchema->numOfCols > 1); return pResult; } @@ -116,8 +114,8 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (pItem->taskInfo) { if (isDeepFree && pItem->tmrId) { - smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId, - i + 1); + smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid, + pItem->tmrId, i + 1); taosTmrStopA(&pItem->tmrId); } tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1); @@ -337,6 +335,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con } pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->suid = suid; + T_REF_INIT_VAL(pRSmaInfo, 1); if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) { goto _err; @@ -392,11 +391,33 @@ int32_t tdProcessRSmaCreate(SSma *pSma, SVCreateStbReq *pReq) { int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { SVnode *pVnode = pSma->pVnode; if (!VND_IS_RSMA(pVnode)) { - smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, + smaTrace("vgId:%d, not drop rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, pReq->suid); return TSDB_CODE_SUCCESS; } + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + + SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pReq->suid); + + if (!pRSmaInfo) { + smaWarn("vgId:%d, drop rsma for stable %s %" PRIi64 " failed no rsma in hash", TD_VID(pVnode), pReq->name, + pReq->suid); + return TSDB_CODE_SUCCESS; + } + + // set del flag for data in mem + RSMA_INFO_SET_DEL(pRSmaInfo); + tdUnRefRSmaInfo(pSma, pRSmaInfo); + + // save to file + smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid); return TSDB_CODE_SUCCESS; } @@ -650,10 +671,10 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType /** * @brief During async commit, the SRSmaInfo object would be COW from iRSmaInfoHash and write lock should be applied. - * - * @param pSma - * @param suid - * @return SRSmaInfo* + * + * @param pSma + * @param suid + * @return SRSmaInfo* */ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); @@ -661,7 +682,6 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { SRSmaInfo *pRSmaInfo = NULL; if (!pEnv) { - // only applicable when rsma env exists return NULL; } @@ -683,18 +703,21 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { SRSmaInfo *pCowRSmaInfo = NULL; // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); - void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - if (iRSmaInfo) { - SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; - if (pIRSmaInfo) { - if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) { - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); - return NULL; - } - if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) { - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - return NULL; + if (!taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t))) { // 2-phase lock + void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + if (iRSmaInfo) { + SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; + if (pIRSmaInfo) { + if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) { + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); + return NULL; + } + smaDebug("vgId:%d, clone rsma info succeed for suid:%" PRIu64, SMA_VID(pSma), suid); + if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) { + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + return NULL; + } } } } @@ -703,21 +726,56 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { return pCowRSmaInfo; } +/** + * @brief During the drop procedure, only need to delete the object in rsmaInfoHash. + * + * @param pSma + * @param suid + * @return SRSmaInfo* + */ +void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = NULL; + SRSmaInfo *pRSmaInfo = NULL; + + if (!pEnv) { + return; + } + + pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + if (!pStat || !RSMA_INFO_HASH(pStat)) { + return; + } + + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + if (pRSmaInfo) { + if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + tdFreeRSmaInfo(pSma, pRSmaInfo, true); + } + taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + smaDebug("vgId:%d, remove from infoHash for table:%" PRIu64 " succeed", SMA_VID(pSma), suid); + } +} + static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid); if (!pRSmaInfo) { - smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); + smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } if (!pRSmaInfo->items[0].taskInfo) { - smaDebug("vgId:%d, return as no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); + smaDebug("vgId:%d, execute rsma, no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } if (inputType == STREAM_INPUT__DATA_SUBMIT) { + tdRefRSmaInfo(pSma, pRSmaInfo); + tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1); tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2); + + tdUnRefRSmaInfo(pSma, pRSmaInfo); } return TSDB_CODE_SUCCESS; @@ -931,10 +989,10 @@ _err: /** * @brief Restore from SRSmaQTaskInfoItem - * - * @param pSma - * @param pItem - * @return int32_t + * + * @param pSma + * @param pItem + * @return int32_t */ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) { SRSmaInfo *pRSmaInfo = NULL; @@ -1271,6 +1329,9 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { } SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + goto _end; + } int8_t fetchTriggerStat = atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); @@ -1279,13 +1340,14 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma), pItem->level, pRSmaInfo->suid); - tdRefSmaStat(pSma, (SSmaStat *)pStat); + // sync procedure => async process + tdRefRSmaInfo(pSma, pRSmaInfo); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false); tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK); - tdUnRefSmaStat(pSma, (SSmaStat *)pStat); + tdUnRefRSmaInfo(pSma, pRSmaInfo); } break; case TASK_TRIGGER_STAT_PAUSED: { smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused", diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index eb17f06cc98f0b7f921fc86dee2b4c2a26fe444d..f18b25bef4793bac3998d3e08ea84f0ec4318e46 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -64,9 +64,16 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa qTaskInfo_t task = pExec->execCol.task[0]; if (qStreamPrepareScan(task, pOffset) < 0) { - ASSERT(pOffset->type == TMQ_OFFSET__LOG); - pRsp->rspOffset = *pOffset; - return 0; + if (pOffset->type == TMQ_OFFSET__LOG) { + pRsp->rspOffset = *pOffset; + return 0; + } else { + tqOffsetResetToLog(pOffset, pHandle->snapshotVer); + if (qStreamPrepareScan(task, pOffset) < 0) { + pRsp->rspOffset = *pOffset; + return 0; + } + } } int32_t rowCnt = 0; @@ -99,7 +106,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa } if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); + tqOffsetResetToLog(pOffset, pHandle->snapshotVer); qStreamPrepareScan(task, pOffset); continue; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 4b2160434fa8aab6a66ee3bed06a572c797aac50..c929c84203d4d085673ce54cd57914d59c152d94 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -243,7 +243,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) void* data = taosMemoryMalloc(msgLen); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - qError("failed to copy data for stream since out of memory"); + tqError("failed to copy data for stream since out of memory"); return -1; } memcpy(data, msg, msgLen); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b49e12fc08d388cc996b6d309bcf354b031f1853..5f730bcfa5aee383f85a53fb115a267b7e641883 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -53,6 +53,7 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { *(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos + 8) = ctime; + vTrace("vgId:%d, table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid); tEndDecode(&dc); } @@ -381,7 +382,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p goto end; } - vInfo("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp); + vDebug("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp); int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids); if (ret != 0) { goto end; @@ -630,6 +631,9 @@ _exit: tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp); tEncoderClear(&ec); + if (vMetaRsp.pSchemas) { + taosMemoryFree(vMetaRsp.pSchemas); + } return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 712cee9fd05d82a5cf72f0842bf5dc8952590c49..dbe44586813e27fba988f57efacfbd0cfce4134a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -17,35 +17,22 @@ #include "vnd.h" static inline bool vnodeIsMsgBlock(tmsg_t type) { - return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA); + return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) || + (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL); } static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; } -static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) { - if (!vnodeIsMsgBlock(type)) return; - - int32_t count = atomic_add_fetch_32(&pVnode->blockCount, 1); - vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type)); -} - -static inline void vnodeWaitBlockMsg(SVnode *pVnode) { - int32_t count = atomic_load_32(&pVnode->blockCount); - if (count <= 0) return; - - vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count); - tsem_wait(&pVnode->syncSem); +static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { + if (vnodeIsMsgBlock(pMsg->msgType)) { + vTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); + tsem_wait(&pVnode->syncSem); + } } -static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) { - if (!vnodeIsMsgBlock(type)) return; - - int32_t count = atomic_load_32(&pVnode->blockCount); - if (count <= 0) return; - - count = atomic_sub_fetch_32(&pVnode->blockCount, 1); - vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type)); - if (count <= 0) { +static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { + if (vnodeIsMsgBlock(pMsg->msgType)) { + vTrace("vgId:%d, msg:%p post block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); tsem_post(&pVnode->syncSem); } } @@ -143,6 +130,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) int32_t code = 0; SRpcMsg *pMsg = NULL; + vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs); + for (int32_t m = 0; m < numOfMsgs; m++) { if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; const STraceId *trace = &pMsg->info.traceId; @@ -165,13 +154,14 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); } + } else if (code == 0) { + vnodeWaitBlockMsg(pVnode, pMsg); + } else { } } } - if (code == 0) { - vnodeAccumBlockMsg(pVnode, pMsg->msgType); - } else if (code < 0) { + if (code < 0) { if (terrno == TSDB_CODE_SYN_NOT_LEADER) { vnodeRedirectRpcMsg(pVnode, pMsg); } else { @@ -182,15 +172,12 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) tmsgSendRsp(&rsp); } } - } else { } vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } - - vnodeWaitBlockMsg(pVnode); } void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { @@ -213,7 +200,7 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { } } - vnodePostBlockMsg(pVnode, pMsg->msgType); + vnodePostBlockMsg(pVnode, pMsg); if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); } @@ -418,7 +405,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon tmsgSendRsp(&rpcMsg); } - vnodePostBlockMsg(pVnode, TDMT_VND_ALTER_REPLICA); + vnodePostBlockMsg(pVnode, pMsg); } static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f9aba30a468c445757fa4cc21e083ba7813767f9..625541e00a1e930359bacdd42c853d287cb77e66 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -15,6 +15,7 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +#include "vnode.h" #include "function.h" #include "nodes.h" #include "plannodes.h" @@ -96,9 +97,9 @@ static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo } void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); -void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); - void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); + +void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); @@ -106,7 +107,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); -int32_t getTableList(void* metaHandle, void* vnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo); +int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); SArray* createSortInfo(SNodeList* pNodeList); SArray* extractPartitionColInfo(SNodeList* pNodeList); SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, @@ -128,4 +129,6 @@ int32_t convertFillType(int32_t mode); int32_t resultrowComparAsc(const void* p1, const void* p2); +int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); + #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 2ffb1be2603fcf12aca305a95f8661310e50b5ad..a7e033e547ebabf9da27f956d47d72071fb9917b 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -311,19 +311,10 @@ typedef struct STableScanInfo { int32_t dataBlockLoadFlag; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; int32_t currentTable; - -#if 0 - struct { - uint64_t uid; - int64_t ts; - } lastStatus; -#endif - - int8_t scanMode; - int8_t noTable; + int8_t scanMode; + int8_t noTable; } STableScanInfo; typedef struct STagScanInfo { @@ -429,8 +420,9 @@ typedef struct SStreamScanInfo { // status for tmq // SSchemaWrapper schema; - STqOffset offset; - + STqOffset offset; + SNode* pTagCond; + SNode* pTagIndexCond; } SStreamScanInfo; typedef struct SSysTableScanInfo { @@ -874,8 +866,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, - STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId); +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, + SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); @@ -966,12 +958,12 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); int32_t createScanTableListInfo(SScanPhysiNode *pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idstr); SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId); + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 17484ec84b14b836ca1942afe8f9aca13872d23b..4c86a46b6cef91efda56ce86324918c16ff24a4b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -72,8 +72,12 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); - taosArrayDestroy(pGroupResInfo->pRows); - pGroupResInfo->pRows = NULL; + for(int32_t i = 0; i < taosArrayGetSize(pGroupResInfo->pRows); ++i) { + SResKeyPos* pRes = taosArrayGetP(pGroupResInfo->pRows, i); + taosMemoryFree(pRes); + } + + pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); pGroupResInfo->index = 0; } @@ -261,7 +265,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) { +int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { SMetaReader mr = {0}; metaReaderInit(&mr, metaHandle, 0); metaGetTableEntryByUid(&mr, info->uid); @@ -276,19 +280,22 @@ static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) { if (TSDB_CODE_SUCCESS != code) { terrno = code; nodesDestroyNode(pTagCondTmp); - return false; + *pQualified = false; + + return code; } ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); SValueNode* pValue = (SValueNode*)pNew; ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL); - bool result = pValue->datum.b; + *pQualified = pValue->datum.b; + nodesDestroyNode(pNew); - return result; + return TSDB_CODE_SUCCESS; } -int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) { +int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo)); @@ -300,8 +307,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, pListInfo->suid = pScanNode->suid; - SNode* pTagCond = (SNode*)pListInfo->pTagCond; - SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond; if (pScanNode->tableType == TSDB_SUPER_TABLE) { if (pTagIndexCond) { SIndexMetaArg metaArg = { @@ -341,9 +346,14 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, int32_t i = 0; while (i < taosArrayGetSize(pListInfo->pTableList)) { STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i); - bool isOk = isTableOk(info, pTagCond, metaHandle); - if (terrno) return terrno; - if (!isOk) { + + bool qualified = true; + code = isTableOk(info, pTagCond, metaHandle, &qualified); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (!qualified) { taosArrayRemove(pListInfo->pTableList, i); continue; } @@ -358,7 +368,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, // put into list as default group, remove it if grouping sorting is required later taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList); - return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index f2008ed97a5c54f1fae090fa721d396ee5eaa49c..90952d578653ac27315a0145ae2b3d8d031ec69a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -153,7 +153,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { return pTaskInfo; } -static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList) { +static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); // let's discard the tables those are not created according to the queried super table. @@ -164,7 +164,7 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons int32_t code = metaGetTableEntryByUid(&mr, *id); if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); + qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr); continue; } @@ -172,6 +172,21 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) { continue; } + + if (pScanInfo->pTagCond != NULL) { + bool qualified = false; + STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid, .lastKey = 0}; + code = isTableOk(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); + continue; + } + + if (!qualified) { + continue; + } + } + /*pScanInfo->pStreamScanOp->pTaskInfo->tableqinfoList.*/ // handle multiple partition @@ -194,7 +209,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo int32_t code = 0; SStreamScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id - SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList); + SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 37d2521e5d800494f0135f9fbdeb20cce87d1cc3..9d3d62cabf06b9b9e756da3ab2acf9cf39a00b63 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -332,6 +332,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); uid = pTableInfo->uid; ts = INT64_MIN; + } else { + return -1; } } /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fe90e58527429de01ef0e57e14fe86c38c12f206..06bb096e59999c3ab4125fa1692239d9bf2c2e6d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "filter.h" #include "function.h" #include "functionMgt.h" @@ -377,6 +376,10 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } +void cleanupExecTimeWindowInfo(SColumnInfoData* pColData) { + colDataDestroy(pColData); +} + void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { @@ -3737,7 +3740,7 @@ void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - taosArrayDestroyEx(pInfo->groupResInfo.pRows, freeItem); + cleanupGroupResInfo(&pInfo->groupResInfo); taosMemoryFreeClear(param); } @@ -4365,8 +4368,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC } SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, - uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, - const char* pUser) { + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* pUser) { int32_t type = nodeType(pPhyNode); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -4374,7 +4376,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, - pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); + pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { pTaskInfo->code = code; return NULL; @@ -4394,7 +4396,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, - pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); + pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { pTaskInfo->code = code; return NULL; @@ -4407,7 +4409,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } SOperatorInfo* pOperator = - createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; @@ -4424,15 +4426,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo }; if (pHandle) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, - pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); + pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { pTaskInfo->code = code; return NULL; } } - SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo, &twSup); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4441,7 +4442,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTableListInfo); + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = terrno; return NULL; @@ -4477,8 +4478,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; - int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, - queryId, taskId); + int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -4502,7 +4502,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES); for (int32_t i = 0; i < size; ++i) { SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); - ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo, pUser); + ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pUser); if (ops[i] == NULL) { return NULL; } @@ -4685,6 +4685,7 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { return pList; } +#if 0 STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, const char* idstr) { int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo); @@ -4718,6 +4719,7 @@ _error: terrno = code; return NULL; } +#endif static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanInfo** ppInfo) { if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -4761,6 +4763,7 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return -1; } +#if 0 int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle, int64_t uid, int64_t ts) { STableScanInfo* pTableScanInfo = NULL; if (extractTbscanInStreamOpTree(pOperator, &pTableScanInfo) < 0) { @@ -4784,6 +4787,7 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa // TODO: set uid and ts to data reader return 0; } +#endif int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) { int32_t code = TDB_CODE_SUCCESS; @@ -4935,10 +4939,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead } (*pTaskInfo)->sql = sql; - (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; - (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; - (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, - &(*pTaskInfo)->tableqinfoList, pPlan->user); + (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user); if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f51dd076a7a035c2875c76bb5c0a12d4d9aee36b..200b920a0ecdacb85e56f2c2bec53ec665b52ba2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1497,9 +1497,8 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pStreamScan); } -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, - SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, - uint64_t taskId) { +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, + SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup) { SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1512,6 +1511,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; + pInfo->pTagCond = pTagCond; + int32_t numOfCols = 0; pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); @@ -2567,26 +2568,19 @@ typedef struct STableMergeScanInfo { int32_t tableEndIndex; bool hasGroupId; uint64_t groupId; + SArray* dataReaders; // array of tsdbReaderT* + SReadHandle readHandle; + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; - SArray* dataReaders; // array of tsdbReaderT* - SReadHandle readHandle; - - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - - SArray* pSortInfo; - SSortHandle* pSortHandle; - - SSDataBlock* pSortInputBlock; - int64_t startTs; // sort start time - - SArray* sortSourceParams; - uint64_t queryId; - uint64_t taskId; + SSDataBlock* pSortInputBlock; + int64_t startTs; // sort start time + SArray* sortSourceParams; SFileBlockLoadRecorder readRecorder; - int64_t numOfRows; - // int32_t prevGroupId; // previous table group id + int64_t numOfRows; SScanInfo scanInfo; int32_t scanTimes; SNode* pFilterNode; // filter info, which is push down by optimizer @@ -2601,26 +2595,25 @@ typedef struct STableMergeScanInfo { SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; SqlFunctionCtx* pPseudoCtx; - // int32_t* rowEntryInfoOffset; SQueryTableDataCond cond; - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan - int32_t dataBlockLoadFlag; - SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time - // window to check if current data block needs to be loaded. - + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + int32_t dataBlockLoadFlag; + // if the upstream is an interval operator, the interval info is also kept here to get the time + // window to check if current data block needs to be loaded. + SInterval interval; SSampleExecInfo sample; // sample execution info } STableMergeScanInfo; int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) { - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTableListInfo); + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idStr) { + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { return code; } if (taosArrayGetSize(pTableListInfo->pTableList) == 0) { - qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId); + qDebug("no table qualified for query, %s" PRIx64, idStr); return TSDB_CODE_SUCCESS; } @@ -3044,8 +3037,7 @@ int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { } SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, - uint64_t taskId) { + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3084,9 +3076,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); - pInfo->queryId = queryId; - pInfo->taskId = taskId; - pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); pInfo->pSortInfo = generateSortByTsInfo(pInfo->pColMatchInfo, pInfo->cond.order); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0fa29fca742cfc1306049438b52dfce9c296912b..026222f83aee909553400f447117d719f8280e54 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -63,7 +63,6 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { - assert(win->skey <= win->ekey); SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId, pTaskInfo, true, pAggSup); @@ -1513,12 +1512,25 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(param); } +static void freeItem(void* param) { + SGroupKeys *pKey = (SGroupKeys*) param; + taosMemoryFree(pKey->pData); +} + void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - taosArrayDestroy(pInfo->pRecycledPages); + pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); + pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); + taosArrayDestroyEx(pInfo->pPrevValues, freeItem); + pInfo->pPrevValues = NULL; + pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + + cleanupGroupResInfo(&pInfo->groupResInfo); + colDataDestroy(&pInfo->twAggSup.timeWindowData); taosMemoryFreeClear(param); } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index aba1df216ed3435c417025807323105013db8977..fcde97ed5f33ae7da7c93a7416597726e26bb90e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -194,8 +194,9 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { void static addTimezoneParam(SNodeList* pList) { char buf[6] = {0}; time_t t = taosTime(NULL); - struct tm* tmInfo = taosLocalTime(&t, NULL); - strftime(buf, sizeof(buf), "%z", tmInfo); + struct tm tmInfo; + taosLocalTime(&t, &tmInfo); + strftime(buf, sizeof(buf), "%z", &tmInfo); int32_t len = (int32_t)strlen(buf); SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); @@ -1422,14 +1423,6 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); - for (int32_t i = 0; i < numOfParams; ++i) { - SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i); - if (QUERY_NODE_VALUE == nodeType(pParamNode)) { - return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); - } - } - pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; return TSDB_CODE_SUCCESS; } @@ -2086,6 +2079,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, .processFunc = topFunction, + .sprocessFunc = topBotScalarFunction, .finalizeFunc = topBotFinalize, .combineFunc = topCombine, .pPartialFunc = "top", @@ -2100,6 +2094,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, .processFunc = bottomFunction, + .sprocessFunc = topBotScalarFunction, .finalizeFunc = topBotFinalize, .combineFunc = bottomCombine, .pPartialFunc = "bottom", @@ -2229,6 +2224,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastRowFunction, + .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize }, { @@ -2249,6 +2245,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = firstFunction, + .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize, .pPartialFunc = "_first_partial", .pMergeFunc = "_first_merge", @@ -2284,6 +2281,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastFunction, + .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize, .pPartialFunc = "_last_partial", .pMergeFunc = "_last_merge", @@ -2331,6 +2329,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, .processFunc = histogramFunction, + .sprocessFunc = histogramScalarFunction, .finalizeFunc = histogramFinalize, .invertFunc = NULL, .combineFunc = histogramCombine, @@ -2463,6 +2462,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getSampleFuncEnv, .initFunc = sampleFunctionSetup, .processFunc = sampleFunction, + .sprocessFunc = sampleScalarFunction, .finalizeFunc = sampleFinalize }, { @@ -2474,6 +2474,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, .processFunc = tailFunction, + .sprocessFunc = tailScalarFunction, .finalizeFunc = NULL }, { @@ -2485,6 +2486,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup, .processFunc = uniqueFunction, + .sprocessFunc = uniqueScalarFunction, .finalizeFunc = NULL }, { @@ -2495,6 +2497,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getModeFuncEnv, .initFunc = modeFunctionSetup, .processFunc = modeFunction, + .sprocessFunc = modeScalarFunction, .finalizeFunc = modeFinalize, }, { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index f9ebde0657ee6794d3358c57865015213c6ab25f..fda02cd9ca387d5ade38721c8eaf0871241abc6b 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -548,37 +548,12 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanLogicPlanTableType = "TableType"; static const char* jkScanLogicPlanTableId = "TableId"; static const char* jkScanLogicPlanStableId = "StableId"; +static const char* jkScanLogicPlanScanType = "ScanType"; static const char* jkScanLogicPlanScanCount = "ScanCount"; static const char* jkScanLogicPlanReverseScanCount = "ReverseScanCount"; static const char* jkScanLogicPlanTagCond = "TagCond"; static const char* jkScanLogicPlanGroupTags = "GroupTags"; -// typedef struct SScanLogicNode { -// uint64_t stableId; -// SVgroupsInfo* pVgroupList; -// EScanType scanType; -// uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count -// STimeWindow scanRange; -// SName tableName; -// bool showRewrite; -// double ratio; -// SNodeList* pDynamicScanFuncs; -// int32_t dataRequired; -// int64_t interval; -// int64_t offset; -// int64_t sliding; -// int8_t intervalUnit; -// int8_t slidingUnit; -// SNode* pTagCond; -// SNode* pTagIndexCond; -// int8_t triggerType; -// int64_t watermark; -// int8_t igExpired; -// SArray* pSmaIndexes; -// SNodeList* pGroupTags; -// bool groupSort; -// } SScanLogicNode; - static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -598,6 +573,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanStableId, pNode->stableId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanType, pNode->scanType); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanCount, pNode->scanSeq[0]); } @@ -634,6 +612,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanStableId, &pNode->stableId); } + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkScanLogicPlanScanType, pNode->scanType, code); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUTinyIntValue(pJson, jkScanLogicPlanScanCount, &pNode->scanSeq[0]); } @@ -1677,6 +1658,7 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) { } static const char* jkProjectPhysiPlanProjections = "Projections"; +static const char* jkProjectPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj; @@ -1685,6 +1667,9 @@ static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkProjectPhysiPlanProjections, pNode->pProjections); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkProjectPhysiPlanMergeDataBlock, pNode->mergeDataBlock); + } return code; } @@ -1696,6 +1681,9 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkProjectPhysiPlanProjections, &pNode->pProjections); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkProjectPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); + } return code; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 68a60e0b35f54a2a12a7cb9b5e1c506f8b6a2a7b..a7c08d8f659c2477c8c28c9711d687176a9477ae 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -201,7 +201,8 @@ static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) { } static bool isUselessCol(SExprNode* pProj) { - if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId)) { + if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId) && + !fmIsPseudoColumnFunc(((SFunctionNode*)pProj)->funcId)) { return false; } return NULL == ((SExprNode*)pProj)->pAssociation; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 19126b28f1b5088b070744203ecdd31b098f32dc..bac073c6316d97be665a352d226a94e79cc637b4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1686,7 +1686,7 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { static bool sysTableFromVnode(const char* pTable) { return (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) || - (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS))); + (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } @@ -5968,6 +5968,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS if (NULL == pReq->tagName) { return TSDB_CODE_OUT_OF_MEMORY; } + pReq->colId = pSchema->colId; SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema); if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) { @@ -6051,6 +6052,7 @@ static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, if (NULL == pReq->colName) { return TSDB_CODE_OUT_OF_MEMORY; } + pReq->colId = pSchema->colId; return TSDB_CODE_SUCCESS; } @@ -6071,6 +6073,7 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt if (NULL == pReq->colName) { return TSDB_CODE_OUT_OF_MEMORY; } + pReq->colId = pSchema->colId; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 4903beddbed2d851e0f6853eb423f58dcaa576f5..9ced5c1cb65e901f7962f1b73b61e417a97dd339 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -20,6 +20,7 @@ typedef struct SLogicPlanContext { SPlanContext* pPlanCxt; SLogicNode* pCurrRoot; + bool hasScan; } SLogicPlanContext; typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, void*, SLogicNode**); @@ -161,6 +162,10 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols return SCAN_TYPE_STREAM; } + if (TSDB_SYSTEM_TABLE == tableType) { + return SCAN_TYPE_SYSTEM_TABLE; + } + if (NULL == pScanCols) { return NULL == pScanPseudoCols ? SCAN_TYPE_TABLE @@ -169,17 +174,6 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols : SCAN_TYPE_TABLE); } - if (TSDB_SYSTEM_TABLE == tableType) { - return SCAN_TYPE_SYSTEM_TABLE; - } - - SNode* pCol = NULL; - FOREACH(pCol, pScanCols) { - if (COLUMN_TYPE_COLUMN == ((SColumnNode*)pCol)->colType) { - return SCAN_TYPE_TABLE; - } - } - return SCAN_TYPE_TABLE; } @@ -300,6 +294,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect nodesDestroyNode((SNode*)pScan); } + pCxt->hasScan = true; + return code; } @@ -1339,9 +1335,9 @@ static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); } -static void setLogicSubplanType(SLogicSubplan* pSubplan) { +static void setLogicSubplanType(bool hasScan, SLogicSubplan* pSubplan) { if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY != nodeType(pSubplan->pNode)) { - pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + pSubplan->subplanType = hasScan ? SUBPLAN_TYPE_SCAN : SUBPLAN_TYPE_MERGE; } else { SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pSubplan->pNode; pSubplan->subplanType = (MODIFY_TABLE_TYPE_INSERT == pModify->modifyType && NULL != pModify->node.pChildren) @@ -1351,7 +1347,7 @@ static void setLogicSubplanType(SLogicSubplan* pSubplan) { } int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { - SLogicPlanContext cxt = {.pPlanCxt = pCxt}; + SLogicPlanContext cxt = {.pPlanCxt = pCxt, .pCurrRoot = NULL, .hasScan = false}; SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { @@ -1364,7 +1360,7 @@ int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, &pSubplan->pNode); if (TSDB_CODE_SUCCESS == code) { setLogicNodeParent(pSubplan->pNode); - setLogicSubplanType(pSubplan); + setLogicSubplanType(cxt.hasScan, pSubplan); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 8250fb462f5e93534495afb4c75f08ee009d65c3..3f619f506f1cb405e61c7027e167eb880836592b 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -35,7 +35,8 @@ typedef struct SPhysiPlanContext { int32_t errCode; int16_t nextDataBlockId; SArray* pLocationHelper; - SArray* pExecNodeList; // SArray + bool hasScan; + bool hasSysScan; } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { @@ -255,7 +256,7 @@ static int32_t addDataBlockSlot(SPhysiPlanContext* pCxt, SNode** pNode, SDataBlo static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true, false); + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, false, false); } static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { @@ -495,8 +496,6 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub return TSDB_CODE_OUT_OF_MEMORY; } vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); - SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; - taosArrayPush(pCxt->pExecNodeList, &node); return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode); } @@ -577,8 +576,6 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan->execNode.nodeId = MNODE_HANDLE; pSubplan->execNode.epSet = pCxt->pPlanCxt->mgmtEpSet; } - SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0}; - taosArrayPush(pCxt->pExecNodeList, &node); if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_DNODE_VARIABLES)) { pScan->mgmtEpSet = pScanLogicNode->pVgroupList->vgroups->epSet; } else { @@ -586,6 +583,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* } tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); + pCxt->hasSysScan = true; return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } @@ -601,6 +599,7 @@ static int32_t createTableMergeScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { + pCxt->hasScan = true; switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: case SCAN_TYPE_BLOCK_INFO: @@ -1806,23 +1805,31 @@ static void setExplainInfo(SPlanContext* pCxt, SQueryPlan* pPlan) { } } +static void setExecNodeList(SPhysiPlanContext* pCxt, SArray* pExecNodeList) { + if (NULL == pExecNodeList) { + return; + } + if (pCxt->hasSysScan || !pCxt->hasScan) { + SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0}; + taosArrayPush(pExecNodeList, &node); + } +} + int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) { SPhysiPlanContext cxt = {.pPlanCxt = pCxt, .errCode = TSDB_CODE_SUCCESS, .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), - .pExecNodeList = pExecNodeList}; + .hasScan = false, + .hasSysScan = false}; if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } - if (QUERY_POLICY_VNODE == tsQueryPolicy) { - taosArrayClear(pExecNodeList); - } - int32_t code = doCreatePhysiPlan(&cxt, pLogicPlan, pPlan); if (TSDB_CODE_SUCCESS == code) { setExplainInfo(pCxt, *pPlan); + setExecNodeList(&cxt, pExecNodeList); } destoryPhysiPlanContext(&cxt); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 7dcba0e1b4f1fb001b15e8638de87f866a760c65..b754c52bbd708765617c6f6cf379aeb29d4dc3f8 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1062,8 +1062,9 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS); } - struct tm *tmInfo = taosLocalTime((const time_t *)&timeVal, NULL); - strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", tmInfo); + struct tm tmInfo; + taosLocalTime((const time_t *)&timeVal, &tmInfo); + strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); int32_t len = (int32_t)strlen(buf); //add timezone string @@ -2601,3 +2602,263 @@ int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SSca pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } + +typedef enum { UNKNOWN_BIN = 0, USER_INPUT_BIN, LINEAR_BIN, LOG_BIN } EHistoBinType; + +static int8_t getHistogramBinType(char* binTypeStr) { + int8_t binType; + if (strcasecmp(binTypeStr, "user_input") == 0) { + binType = USER_INPUT_BIN; + } else if (strcasecmp(binTypeStr, "linear_bin") == 0) { + binType = LINEAR_BIN; + } else if (strcasecmp(binTypeStr, "log_bin") == 0) { + binType = LOG_BIN; + } else { + binType = UNKNOWN_BIN; + } + + return binType; +} + +typedef struct SHistoFuncBin { + double lower; + double upper; + int64_t count; + double percentage; +} SHistoFuncBin; + +static bool getHistogramBinDesc(SHistoFuncBin** bins, int32_t* binNum, char* binDescStr, int8_t binType, bool normalized) { + cJSON* binDesc = cJSON_Parse(binDescStr); + int32_t numOfBins; + double* intervals; + if (cJSON_IsObject(binDesc)) { /* linaer/log bins */ + int32_t numOfParams = cJSON_GetArraySize(binDesc); + int32_t startIndex; + if (numOfParams != 4) { + return false; + } + + cJSON* start = cJSON_GetObjectItem(binDesc, "start"); + cJSON* factor = cJSON_GetObjectItem(binDesc, "factor"); + cJSON* width = cJSON_GetObjectItem(binDesc, "width"); + cJSON* count = cJSON_GetObjectItem(binDesc, "count"); + cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity"); + + if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { + return false; + } + + if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 + return false; + } + + if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || + (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { + return false; + } + + int32_t counter = (int32_t)count->valueint; + if (infinity->valueint == false) { + startIndex = 0; + numOfBins = counter + 1; + } else { + startIndex = 1; + numOfBins = counter + 3; + } + + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { + // linear bin process + if (width->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble + i * width->valuedouble; + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) { + // log bin process + if (start->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else { + taosMemoryFree(intervals); + return false; + } + + if (infinity->valueint == true) { + intervals[0] = -INFINITY; + intervals[numOfBins - 1] = INFINITY; + // in case of desc bin orders, -inf/inf should be swapped + ASSERT(numOfBins >= 4); + if (intervals[1] > intervals[numOfBins - 2]) { + TSWAP(intervals[0], intervals[numOfBins - 1]); + } + } + } else if (cJSON_IsArray(binDesc)) { /* user input bins */ + if (binType != USER_INPUT_BIN) { + return false; + } + numOfBins = cJSON_GetArraySize(binDesc); + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + cJSON* bin = binDesc->child; + if (bin == NULL) { + taosMemoryFree(intervals); + return false; + } + int i = 0; + while (bin) { + intervals[i] = bin->valuedouble; + if (!cJSON_IsNumber(bin)) { + taosMemoryFree(intervals); + return false; + } + if (i != 0 && intervals[i] <= intervals[i - 1]) { + taosMemoryFree(intervals); + return false; + } + bin = bin->next; + i++; + } + } else { + return false; + } + + *binNum = numOfBins - 1; + *bins = taosMemoryCalloc(numOfBins, sizeof(SHistoFuncBin)); + for (int32_t i = 0; i < *binNum; ++i) { + (*bins)[i].lower = intervals[i] < intervals[i + 1] ? intervals[i] : intervals[i + 1]; + (*bins)[i].upper = intervals[i + 1] > intervals[i] ? intervals[i + 1] : intervals[i]; + (*bins)[i].count = 0; + } + + taosMemoryFree(intervals); + return true; +} + +int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + SHistoFuncBin *bins; + int32_t numOfBins = 0; + int32_t totalCount = 0; + + int8_t binType = getHistogramBinType(varDataVal(pInput[1].columnData->pData)); + char* binDesc = varDataVal(pInput[2].columnData->pData); + int64_t normalized = *(int64_t *)(pInput[3].columnData->pData); + + int32_t type = GET_PARAM_TYPE(pInput); + if (!getHistogramBinDesc(&bins, &numOfBins, binDesc, binType, (bool)normalized)) { + return TSDB_CODE_FAILED; + } + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + char* data = colDataGetData(pInputData, i); + double v; + GET_TYPED_DATA(v, double, type, data); + + for (int32_t k = 0; k < numOfBins; ++k) { + if (v > bins[k].lower && v <= bins[k].upper) { + bins[k].count++; + totalCount++; + break; + } + } + } + + if (normalized) { + for (int32_t k = 0; k < numOfBins; ++k) { + if (totalCount != 0) { + bins[k].percentage = bins[k].count / (double)totalCount; + } else { + bins[k].percentage = 0; + } + } + } + + for (int32_t k = 0; k < numOfBins; ++k) { + int32_t len; + char buf[512] = {0}; + if (!normalized) { + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}", + bins[k].lower, bins[k].upper, bins[k].count); + } else { + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", + bins[k].lower, bins[k].upper, bins[k].percentage); + } + varDataSetLen(buf, len); + colDataAppend(pOutputData, k, buf, false); + } + + taosMemoryFree(bins); + pOutput->numOfRows = numOfBins; + return TSDB_CODE_SUCCESS; +} + +int32_t selectScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int32_t type = GET_PARAM_TYPE(pInput); + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, 0); + continue; + } + + char* data = colDataGetData(pInputData, i); + colDataAppend(pOutputData, i, data, false); + } + + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} + +int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index f6eb9e32f28e18a75379ea0d789d9246f67fabe2..b59a8126786db7f0db18a3bbc86a0e8c734abfee 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -143,76 +143,80 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { } static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { - int32_t cnt = 0; - void* data = NULL; while (1) { - SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); - if (qItem == NULL) { - qDebug("stream exec over, queue empty"); - break; - } - if (data == NULL) { - data = qItem; - if (qItem->type == STREAM_INPUT__DATA_BLOCK) { - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ - } - streamQueueProcessSuccess(pTask->inputQueue); - } else { - if (streamAppendQueueItem(data, qItem) < 0) { - streamQueueProcessFail(pTask->inputQueue); + int32_t cnt = 0; + void* data = NULL; + while (1) { + SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); + if (qItem == NULL) { + qDebug("stream exec over, queue empty"); break; - } else { - cnt++; - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + } + if (data == NULL) { + data = qItem; streamQueueProcessSuccess(pTask->inputQueue); - taosArrayDestroy(((SStreamDataBlock*)qItem)->blocks); - taosFreeQitem(qItem); + if (qItem->type == STREAM_INPUT__DATA_BLOCK) { + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + } else { + break; + } + } else { + if (streamAppendQueueItem(data, qItem) < 0) { + streamQueueProcessFail(pTask->inputQueue); + break; + } else { + cnt++; + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + streamQueueProcessSuccess(pTask->inputQueue); + taosArrayDestroy(((SStreamDataBlock*)qItem)->blocks); + taosFreeQitem(qItem); + } } } - } - if (pTask->taskStatus == TASK_STATUS__DROPPING) { - if (data) streamFreeQitem(data); - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return NULL; - } + if (pTask->taskStatus == TASK_STATUS__DROPPING) { + if (data) streamFreeQitem(data); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return NULL; + } - if (data == NULL) return pRes; + if (data == NULL) break; - if (pTask->execType == TASK_EXEC__NONE) { - ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); - streamTaskOutput(pTask, data); - return pRes; - } + if (pTask->execType == TASK_EXEC__NONE) { + ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); + streamTaskOutput(pTask, data); + return pRes; + } - qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); - streamTaskExecImpl(pTask, data, pRes); - qDebug("stream task %d exec end", pTask->taskId); + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); + streamTaskExecImpl(pTask, data, pRes); + qDebug("stream task %d exec end", pTask->taskId); - if (taosArrayGetSize(pRes) != 0) { - SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); - if (qRes == NULL) { - streamQueueProcessFail(pTask->inputQueue); - taosArrayDestroy(pRes); - return NULL; - } - qRes->type = STREAM_INPUT__DATA_BLOCK; - qRes->blocks = pRes; - if (streamTaskOutput(pTask, qRes) < 0) { - /*streamQueueProcessFail(pTask->inputQueue);*/ - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - taosFreeQitem(qRes); - return NULL; - } - if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; - qRes->childId = pTask->selfChildId; - qRes->sourceVer = pSubmit->ver; + if (taosArrayGetSize(pRes) != 0) { + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + if (qRes == NULL) { + streamQueueProcessFail(pTask->inputQueue); + taosArrayDestroy(pRes); + return NULL; + } + qRes->type = STREAM_INPUT__DATA_BLOCK; + qRes->blocks = pRes; + if (streamTaskOutput(pTask, qRes) < 0) { + /*streamQueueProcessFail(pTask->inputQueue);*/ + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); + return NULL; + } + if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + qRes->childId = pTask->selfChildId; + qRes->sourceVer = pSubmit->ver; + } + /*streamQueueProcessSuccess(pTask->inputQueue);*/ + pRes = taosArrayInit(0, sizeof(SSDataBlock)); } - /*streamQueueProcessSuccess(pTask->inputQueue);*/ - pRes = taosArrayInit(0, sizeof(SSDataBlock)); - } - streamFreeQitem(data); + streamFreeQitem(data); + } return pRes; } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0cbd7d36b2233214944bdaa9c08283316e093e7a..3a30cf801e670db0f7c786f8e11114e9d9017599 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -192,6 +192,7 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode); int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); +int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode); // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index bb7454ea6ff84c499d8e830808f2fbd731f70bfa..ddb2b9355e1540bc1b52d3fe9ddb9d3546e57bf8 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1298,6 +1298,12 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { return ret; } +int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) { + syncNodeStopHeartbeatTimer(pSyncNode); + syncNodeStartHeartbeatTimer(pSyncNode); + return 0; +} + // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index cca451ceb7861ab440c5244754a5b94c8c1cbdf5..6e8fd326414234d51514a893f0a2dcea29572af0 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -103,10 +103,9 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg // if pgno == 0 fetch new btree root leaf page if (pgno == 0) { // fetch page & insert into main db - // allocate a new child page SPage *pPage; TXN txn; - tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); + tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); pPager->inTran = 1; @@ -118,8 +117,6 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg return -1; } - // TODO: Need to zero the page - ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { return -1; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index e7765ed6673ca7be767766b705b5ceb70f2648a3..d9a44ba5705087d83ab5c0ec5b5b56177fd6dd73 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -473,12 +473,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { return -1; } - TXN txn; - tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - SBtreeInitPageArg iArg; - iArg.pBt = pBt; - iArg.flags = 0; - for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) { // read pgno & the page from journal SPgno pgno; @@ -494,20 +488,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { return -1; } - /* - ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &iArg, &txn); - if (ret < 0) { - return -1; - } - - // write the page to db - ret = tdbPagerWritePageToDB(pPager, pPage); - if (ret < 0) { - return -1; - } - - tdbPCacheRelease(pPager->pCache, pPage, &txn); - */ i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { ASSERT(0); @@ -523,8 +503,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { tdbOsFSync(pPager->fd); - tdbTxnClose(&txn); - tdbOsFree(pageBuf); tdbOsClose(jfd); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 908523f2a66f50c081152752fc5cafb28b8eebae..c47964803a812c3013b14b44c374496799b440b1 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -483,6 +483,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) { pRead->pHead->head.version, ver); pRead->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); return -1; } @@ -491,6 +492,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) { wError("vgId:%d, unexpected wal log index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); pRead->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); return -1; } pRead->curVersion++; diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 7e6e5088177c9d157041211f77368f58fb6a37ee..0cb4228e427c14c0e708f113e263e6b619e9bd3c 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -357,14 +357,88 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { time_t taosTime(time_t *t) { return time(t); } -time_t taosMktime(struct tm *timep) { return mktime(timep); } +time_t taosMktime(struct tm *timep) { +#ifdef WINDOWS + struct tm tm1 = {0}; + LARGE_INTEGER t; + FILETIME f; + SYSTEMTIME s; + FILETIME ff; + SYSTEMTIME ss; + LARGE_INTEGER offset; + + time_t tt = 0; + localtime_s(&tm1, &tt); + ss.wYear = tm1.tm_year + 1900; + ss.wMonth = tm1.tm_mon + 1; + ss.wDay = tm1.tm_wday; + ss.wHour = tm1.tm_hour; + ss.wMinute = tm1.tm_min; + ss.wSecond = tm1.tm_sec; + ss.wMilliseconds = 0; + SystemTimeToFileTime(&ss, &ff); + offset.QuadPart = ff.dwHighDateTime; + offset.QuadPart <<= 32; + offset.QuadPart |= ff.dwLowDateTime; + + s.wYear = timep->tm_year + 1900; + s.wMonth = timep->tm_mon + 1; + s.wDay = timep->tm_wday; + s.wHour = timep->tm_hour; + s.wMinute = timep->tm_min; + s.wSecond = timep->tm_sec; + s.wMilliseconds = 0; + SystemTimeToFileTime(&s, &f); + t.QuadPart = f.dwHighDateTime; + t.QuadPart <<= 32; + t.QuadPart |= f.dwLowDateTime; + + t.QuadPart -= offset.QuadPart; + return (time_t)(t.QuadPart / 10000000); +#else + return mktime(timep); +#endif + } struct tm *taosLocalTime(const time_t *timep, struct tm *result) { if (result == NULL) { return localtime(timep); } #ifdef WINDOWS - localtime_s(result, timep); + if (*timep < 0) { + SYSTEMTIME ss,s; + FILETIME ff,f; + LARGE_INTEGER offset; + struct tm tm1; + time_t tt = 0; + localtime_s(&tm1, &tt); + ss.wYear = tm1.tm_year + 1900; + ss.wMonth = tm1.tm_mon + 1; + ss.wDay = tm1.tm_mday; + ss.wHour = tm1.tm_hour; + ss.wMinute = tm1.tm_min; + ss.wSecond = tm1.tm_sec; + ss.wMilliseconds = 0; + SystemTimeToFileTime(&ss, &ff); + offset.QuadPart = ff.dwHighDateTime; + offset.QuadPart <<= 32; + offset.QuadPart |= ff.dwLowDateTime; + offset.QuadPart += *timep * 10000000; + f.dwLowDateTime = offset.QuadPart & 0xffffffff; + f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff; + FileTimeToSystemTime(&f, &s); + result->tm_sec = s.wSecond; + result->tm_min = s.wMinute; + result->tm_hour = s.wHour; + result->tm_mday = s.wDay; + result->tm_mon = s.wMonth-1; + result->tm_year = s.wYear-1900; + result->tm_wday = s.wDayOfWeek; + result->tm_yday = 0; + result->tm_isdst = 0; + } else { + localtime_s(result, timep); + } #else localtime_r(timep, result); #endif diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 08155f656bfb313dc1305eb1d0c846ee98f57cb9..8f0bfdd4811675f5b2a68f31d53fbce3d19c491c 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -30,6 +30,7 @@ import argparse import sys import os import io +import datetime import signal import traceback import requests @@ -1107,14 +1108,20 @@ class Database: # TODO: fix the error as result of above: "tsdb timestamp is out of range" @classmethod def setupLastTick(cls): - t1 = datetime.datetime(2020, 6, 1) + # start time will be auto generated , start at 10 years ago local time + local_time = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-16] + local_epoch_time = [int(i) for i in local_time.split("-")] + #local_epoch_time will be such as : [2022, 7, 18] + + t1 = datetime.datetime(local_epoch_time[0]-5, local_epoch_time[1], local_epoch_time[2]) t2 = datetime.datetime.now() # maybe a very large number, takes 69 years to exceed Python int range elSec = int(t2.timestamp() - t1.timestamp()) elSec2 = (elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500)) * \ 500 # a number representing seconds within 10 years # print("elSec = {}".format(elSec)) - t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years + + t3 = datetime.datetime(local_epoch_time[0]-10, local_epoch_time[1], local_epoch_time[2]) # default "keep" is 10 years t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above Logging.debug("Setting up TICKS to start from: {}".format(t4)) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0bdbd644a0ef400b1f4110ea345cef3023f63ef9..bff6177ad2e3e01185fc11813f4363c180480b62 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -312,6 +312,7 @@ ./test.sh -f tsim/valgrind/checkError3.sim ./test.sh -f tsim/valgrind/checkError4.sim ./test.sh -f tsim/valgrind/checkError5.sim +./test.sh -f tsim/valgrind/checkError6.sim # --- vnode # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index e9a295820c611089bf56d0cb47dcc189f9c3f0a9..1823f182c98673b981072d4612c56d46636bfb06 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -111,6 +111,15 @@ if $hasleader != 1 then goto step2 endi +# sql use db; +# sql create table stb (ts timestamp, c int) tags (t int); +# sql create table t0 using stb tags (0); +# sql insert into t0 values(now, 1); +# sql show db.stables; +# sql show db.tables; +# sql show db.vgroups; +return + sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd" sql create table db.ctb using db.stb tags(101, "102") sql insert into db.ctb values(now, 1, "2") diff --git a/tests/script/tsim/valgrind/checkError5.sim b/tests/script/tsim/valgrind/checkError5.sim index 61964d1c42b3da2a545dd32877f13d108e2f61ef..f3d418cfd16da10ed15dc39ccb99ea8ac430446d 100644 --- a/tests/script/tsim/valgrind/checkError5.sim +++ b/tests/script/tsim/valgrind/checkError5.sim @@ -26,6 +26,7 @@ print =============== step2: create db sql create database db sql use db sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" +sql create table db.c1 using db.stb tags(101, 102, "103") print =============== step3: alter stb sql_error alter table db.stb add column ts int @@ -42,9 +43,8 @@ sql alter table db.stb drop tag c1 sql alter table db.stb drop tag t5 sql alter table db.stb MODIFY tag t3 binary(32) sql alter table db.stb rename tag t1 tx - sql alter table db.stb comment 'abcde' ; -goto _OVER +sql drop table db.stb print =============== step4: alter tb sql create table tb (ts timestamp, a int) @@ -66,6 +66,35 @@ sql alter table tb add column h binary(10) sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb sql select * from tb order by ts desc +print =============== step5: alter stb and insert data +sql create table stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" +sql show db.stables +sql describe stb +sql_error alter table stb add column ts int + +sql create table db.ctb using db.stb tags(101, 102, "103") +sql insert into db.ctb values(now, 1, "2") +sql show db.tables +sql select * from db.stb +sql select * from tb + +sql alter table stb add column c3 int +sql describe stb +sql select * from db.stb +sql select * from tb +sql insert into db.ctb values(now+1s, 1, 2, 3) +sql select * from db.stb + +sql alter table db.stb add column c4 bigint +sql select * from db.stb +sql insert into db.ctb values(now+2s, 1, 2, 3, 4) + +sql alter table db.stb drop column c1 +sql reset query cache +sql select * from tb +sql insert into db.ctb values(now+3s, 2, 3, 4) +sql select * from db.stb + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index a9f66647f9250ef9acb0707eaa269f3c7a524c8c..2783e9477191627872f6cf52ee06c167b531d7e6 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -68,7 +68,7 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 0 then +if $system_content > 3 then return -1 endi diff --git a/tests/system-test/0-others/cachelast.py b/tests/system-test/0-others/cachemodel.py similarity index 62% rename from tests/system-test/0-others/cachelast.py rename to tests/system-test/0-others/cachemodel.py index 2df6b8c9ccb9ba3bb342ca874ab7b10f0971c569..09538e6678f16d7eebbafce45fad6046e551a6bc 100644 --- a/tests/system-test/0-others/cachelast.py +++ b/tests/system-test/0-others/cachemodel.py @@ -56,6 +56,16 @@ class TDTestCase: } return numbers.get(value, 'other') + def getCacheModelNum(self,str): + numbers = { + "none" : 0, + "last_row" : 1, + "last_value" : 2, + "both" : 3 + + } + return numbers.get(str, 'other') + def prepare_datas(self): for i in range(4): str = self.getCacheModelStr(i) @@ -69,7 +79,7 @@ class TDTestCase: tdSql.execute(" insert into tb1 values(now , %d, %f)" %(k,k*10) ) tdSql.execute(" insert into tb2 values(now , %d, %f)" %(k,k*10) ) - def check_cache_last_sets(self): + def check_cachemodel_sets(self): # check cache_last value for database @@ -84,52 +94,54 @@ class TDTestCase: # print(cache_last_value) if dbname in ["information_schema" , "performance_schema"]: continue - cache_lasts[dbname]=cache_last_value + cache_lasts[dbname]=self.getCacheModelNum(cache_last_value) # cache_last_set value for k , v in cache_lasts.items(): - if k=="testdb_"+str(v): - tdLog.info(" database %s cache_last value check pass, value is %s "%(k,v) ) + if k=="testdb_"+str(self.getCacheModelStr(v)): + tdLog.info(" database %s cache_last value check pass, value is %s "%(k,self.getCacheModelStr(v)) ) else: - tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,v) ) + tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,self.getCacheModelStr(v)) ) # # check storage layer implementation - # buildPath = self.getBuildPath() - # if (buildPath == ""): - # tdLog.exit("taosd not found!") - # else: - # tdLog.info("taosd found in %s" % buildPath) - # dataPath = buildPath + "/../sim/dnode1/data" - # abs_vnodePath = os.path.abspath(dataPath)+"/vnode/" - # tdLog.info("abs_vnodePath: %s" % abs_vnodePath) + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + dataPath = buildPath + "/../sim/dnode1/data" + abs_vnodePath = os.path.abspath(dataPath)+"/vnode/" + tdLog.info("abs_vnodePath: %s" % abs_vnodePath) - # tdSql.query(" show dnodes ") - # dnode_id = tdSql.queryResult[0][0] - - # for dbname in cache_lasts.keys(): - # print(dbname) - # tdSql.execute(" use %s" % dbname) - # tdSql.query(" show vgroups ") - # vgroups_infos = tdSql.queryResult - # for vgroup_info in vgroups_infos: - # vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json" - # vnode_info_of_db = f"cat {vnode_json}" - # vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8") - # infoDict = json.loads(vnode_info) - # vnode_json_of_dbname = f"{dnode_id}."+ dbname - # config = infoDict["config"] - # if infoDict["config"]["dbname"] == vnode_json_of_dbname: - # if "cachelast" in infoDict["config"]: - # if int(infoDict["config"]["cachelast"]) != cache_lasts[dbname]: - # tdLog.exit("cachelast value is error in vnode.json of vnode%d "%(vgroup_info[0])) - # else: - # tdLog.exit("cachelast not found in vnode.json of vnode%d "%(vgroup_info[0])) - - def restart_check_cache_last_sets(self): + tdSql.query(" show dnodes ") + dnode_id = tdSql.queryResult[0][0] + + for dbname in cache_lasts.keys(): + # print(dbname) + tdSql.execute(" use %s" % dbname) + tdSql.query(" show vgroups ") + vgroups_infos = tdSql.queryResult + for vgroup_info in vgroups_infos: + vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json" + vnode_info_of_db = f"cat {vnode_json}" + vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8") + infoDict = json.loads(vnode_info) + vnode_json_of_dbname = f"{dnode_id}."+ dbname + config = infoDict["config"] + if infoDict["config"]["dbname"] == vnode_json_of_dbname: + if "cacheLast" in infoDict["config"]: + if int(infoDict["config"]["cacheLast"]) != cache_lasts[dbname]: + tdLog.exit("cachemodel value is error in vnode.json of vnode%d "%(vgroup_info[0])) + else: + tdLog.info("cachemodel value is success in vnode.json of vnode%d "%(vgroup_info[0])) + else: + tdLog.exit("cacheLast not found in vnode.json of vnode%d "%(vgroup_info[0])) + + def restart_check_cachemodel_sets(self): for i in range(3): tdSql.query("show dnodes") @@ -137,14 +149,14 @@ class TDTestCase: tdDnodes.stop(index) tdDnodes.start(index) time.sleep(3) - self.check_cache_last_sets() + self.check_cachemodel_sets() def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring self.illegal_params() self.prepare_datas() - self.check_cache_last_sets() - self.restart_check_cache_last_sets() + self.check_cachemodel_sets() + self.restart_check_cachemodel_sets() def stop(self): tdSql.close() diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py new file mode 100644 index 0000000000000000000000000000000000000000..a7eba2d97d6f56d177f591935a91b73d41654fa3 --- /dev/null +++ b/tests/system-test/1-insert/delete_data.py @@ -0,0 +1,190 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string + +from numpy import logspace +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.rowNum = 10 + self.tbnum = 20 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.str_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', + + } + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,100)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.base_data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + def insert_base_data(self,col_type,tbname,rows,base_data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') + + def delete_all_data(self,tbname,col_type,row_num,base_data,dbname): + tdSql.execute(f'delete from {tbname}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(0) + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(row_num) + def delete_one_row(self,tbname,column_type,column_name,base_data,dbname): + tdSql.execute(f'delete from {tbname} where ts={self.ts}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {column_name} from {tbname}') + tdSql.checkRows(self.rowNum-1) + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + tdSql.checkRows(0) + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})') + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + if column_type.lower() == 'float' or column_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{column_type} data check failure') + elif 'binary' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) + elif 'nchar' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) + + def delete_rows(self): + + + pass + def delete_error(self,tbname,column_name,column_type,base_data): + for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: + if 'binary' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') + elif 'nchar' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') + else: + tdSql.error('delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') + + def delete_data_ntb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) + self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.dbname) + self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname) + self.delete_error(self.ntbname,col_name,col_type,self.base_data) + for i in range(self.rowNum): + tdSql.execute(f'delete from {self.ntbname} where ts>{self.ts+i}') + tdSql.execute(f'flush database {self.dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {self.ntbname}') + tdSql.checkRows(i+1) + self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) + + tdSql.execute(f'drop table {self.ntbname}') + + def run(self): + self.delete_data_ntb() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py new file mode 100644 index 0000000000000000000000000000000000000000..27e1559d7e0411d7bc11208b4430d590f89e9ad8 --- /dev/null +++ b/tests/system-test/1-insert/update_data.py @@ -0,0 +1,229 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.setsql = TDSetSql() + self.dbname = 'db_test' + self.ntbname = 'ntb' + self.stbname = 'stb' + self.ctbname = 'ctb' + self.ts = 1537146000000 + self.str_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', + 'col_ts' : 'timestamp' + } + + def data_check(self,tbname,col_name,col_type,value): + tdSql.query(f'select {col_name} from {tbname}') + if col_type.lower() == 'float' or col_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - value) / value <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{col_name} data check failure') + elif col_type.lower() == 'timestamp': + tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f"))) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],value) + def update_and_check_data(self,tbname,col_name,col_type,value,dbname): + if 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{value})') + self.data_check(tbname,col_name,col_type,value) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + self.data_check(tbname,col_name,col_type,value) + for func in ['first','last']: + tdSql.execute(f'select {func}({col_name}) from {tbname}') + def error_check(self,tbname,column_dict,tb_type=None,stbname=None): + str_length = self.str_length+1 + for col_name,col_type in column_dict.items(): + if tb_type == 'ntb': + tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})') + elif tb_type == 'ctb': + tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') + tdSql.execute(f'create table {tbname} using {stbname} tags(1)') + tdSql.execute(f'insert into {tbname} values({self.ts},null)') + if col_type.lower() == 'double': + for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'float': + for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + for error_value in [tdCom.getLongName(str_length)]: + tdSql.error(f'insert into {tbname} values({self.ts},"{error_value}")') + elif col_type.lower() == 'bool': + for error_value in [tdCom.getLongName(self.str_length)]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'tinyint': + for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'smallint': + for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'int': + for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bigint': + for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'tinyint unsigned': + for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'smallint unsigned': + for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'int unsigned': + for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bigint unsigned': + for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + tdSql.execute(f'drop table {tbname}') + if tb_type == 'ctb': + tdSql.execute(f'drop table {stbname}') + def update_data_check(self,tbname,column_dict,dbname,tb_type=None,stbname=None): + up_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + up_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + up_int = random.randint(constant.INT_MIN,constant.INT_MAX) + up_bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + up_untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + up_unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + up_unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + up_unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + up_bool = random.randint(0,100)%2 + up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + binary_length = random.randint(0,self.str_length) + nchar_length = random.randint(0,self.str_length) + up_binary = tdCom.getLongName(binary_length) + up_nchar = tdCom.getLongName(nchar_length) + for col_name,col_type in column_dict.items(): + if tb_type == 'ntb': + tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})') + elif tb_type == 'ctb': + tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') + tdSql.execute(f'create table {tbname} using {stbname} tags(1)') + tdSql.execute(f'insert into {tbname} values({self.ts},null)') + if col_type.lower() == 'tinyint': + self.update_and_check_data(tbname,col_name,col_type,up_tinyint,dbname) + elif col_type.lower() == 'smallint': + self.update_and_check_data(tbname,col_name,col_type,up_smallint,dbname) + elif col_type.lower() == 'int': + self.update_and_check_data(tbname,col_name,col_type,up_int,dbname) + elif col_type.lower() == 'bigint': + self.update_and_check_data(tbname,col_name,col_type,up_bigint,dbname) + elif col_type.lower() == 'tinyint unsigned': + self.update_and_check_data(tbname,col_name,col_type,up_untinyint,dbname) + elif col_type.lower() == 'smallint unsigned': + self.update_and_check_data(tbname,col_name,col_type,up_unsmallint,dbname) + elif col_type.lower() == 'int unsigned': + self.update_and_check_data(tbname,col_name,col_type,up_unint,dbname) + elif col_type.lower() == 'bigint unsigned': + self.update_and_check_data(tbname,col_name,col_type,up_unbigint,dbname) + elif col_type.lower() == 'bool': + self.update_and_check_data(tbname,col_name,col_type,up_bool,dbname) + elif col_type.lower() == 'float': + self.update_and_check_data(tbname,col_name,col_type,up_float,dbname) + elif col_type.lower() == 'double': + self.update_and_check_data(tbname,col_name,col_type,up_double,dbname) + elif 'binary' in col_type.lower(): + self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname) + elif 'nchar' in col_type.lower(): + self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname) + elif col_type.lower() == 'timestamp': + self.update_and_check_data(tbname,col_name,col_type,self.ts+1,dbname) + tdSql.execute(f'insert into {tbname} values({self.ts},null)') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkEqual(tdSql.queryResult[0][0],None) + tdSql.execute(f'flush database {self.dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkEqual(tdSql.queryResult[0][0],None) + tdSql.execute(f'drop table {tbname}') + if tb_type == 'ctb': + tdSql.execute(f'drop table {stbname}') + def update_check(self): + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + self.update_data_check(self.ntbname,self.column_dict,self.dbname,'ntb') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)') + if 'binary' in col_type.lower(): + up_binary = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.ntbname} modify column {col_name} binary({self.str_length+1})') + self.update_and_check_data(self.ntbname,col_name,col_type,up_binary,self.dbname) + elif 'nchar' in col_type.lower(): + up_nchar = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.ntbname} modify column {col_name} nchar({self.str_length+1})') + self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname) + tdSql.execute(f'drop table {self.ntbname}') + self.update_data_check(self.ctbname,self.column_dict,self.dbname,'ctb',self.stbname) + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') + tdSql.execute(f'create table {self.ctbname} using {self.stbname} tags(1)') + tdSql.execute(f'insert into {self.ctbname} values({self.ts},null)') + if 'binary' in col_type.lower(): + up_binary = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.stbname} modify column {col_name} binary({self.str_length+1})') + self.update_and_check_data(self.ctbname,col_name,col_type,up_binary,self.dbname) + elif 'nchar' in col_type.lower(): + up_nchar = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.stbname} modify column {col_name} nchar({self.str_length+1})') + self.update_and_check_data(self.ctbname,col_name,col_type,up_nchar,self.dbname) + tdSql.execute(f'drop table {self.stbname}') + + def update_check_error(self): + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + self.error_check(self.ntbname,self.column_dict,'ntb') + self.error_check(self.ctbname,self.column_dict,'ctb',self.stbname) + + def run(self): + self.update_check() + self.update_check_error() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index fce3cc70432fa694a10e288c945cbffc43c35667..c79e3b58fbd0760d9e5a55caaea118df299a6f11 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -22,15 +22,16 @@ class TDTestCase: self.ts = 1434938400000 self.time_step = 1000 - def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): + def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ,cache_value ): + tdSql.execute("drop database if exists test ") tdLog.info(" prepare datas for auto check abs function ") - tdSql.execute(" create database test cachemodel 'last_row' ") + tdSql.execute(f" create database test cachemodel {cache_value} ") tdSql.execute(" use test ") - tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ + tdSql.execute(" create stable test.stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") for tbnum in range(tbnums): - tbname = "sub_tb_%d"%tbnum + tbname = "test.sub_tb_%d"%tbnum tdSql.execute(" create table %s using stb tags(%d) "%(tbname , tbnum)) ts = self.ts @@ -47,7 +48,7 @@ class TDTestCase: c9 = "'nchar_val'" c10 = ts tdSql.execute(f" insert into {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})") - + tdSql.execute("use test") tbnames = ["stb", "sub_tb_1"] support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"] @@ -56,17 +57,19 @@ class TDTestCase: coltypes = tdSql.queryResult for coltype in coltypes: colname = coltype[0] - abs_sql = "select abs({}) from {} order by tbname ".format(colname, tbname) - origin_sql = "select {} from {} order by tbname".format(colname, tbname) + abs_sql = "select abs({}) from {} order by tbname ".format(colname, 'test.'+tbname) + origin_sql = "select {} from {} order by tbname".format(colname, 'test.'+tbname) if coltype[1] in support_types: self.check_result_auto(origin_sql , abs_sql) - - def prepare_datas(self): - tdSql.execute("create database if not exists db keep 3650 duration 1000 cachemodel 'last_row'") + def prepare_datas(self ,cache_value): + tdSql.execute("drop database if exists db ") + create_db_sql = f"create database if not exists db keep 3650 duration 1000 cachemodel {cache_value}" + tdSql.execute(create_db_sql) + tdSql.execute("use db") tdSql.execute( - '''create table stb1 + '''create table db.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int) ''' @@ -74,38 +77,38 @@ class TDTestCase: tdSql.execute( ''' - create table t1 + create table db.t1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) ''' ) for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + tdSql.execute(f'create table db.ct{i+1} using stb1 tags ( {i+1} )') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into db.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into db.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + "insert into db.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") tdSql.execute( - "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + "insert into db.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") tdSql.execute( - "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + "insert into db.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") tdSql.execute( - "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + "insert into db.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") tdSql.execute( - "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + "insert into db.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( - "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + "insert into db.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( - "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + "insert into db.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( - f'''insert into t1 values + f'''insert into db.t1 values ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) @@ -121,51 +124,53 @@ class TDTestCase: ''' ) - def prepare_tag_datas(self): + def prepare_tag_datas(self,cache_value): + + tdSql.execute("drop database if exists testdb ") # prepare datas - tdSql.execute( - "create database if not exists testdb keep 3650 duration 1000 cachemodel 'last_row'") + tdSql.execute(f"create database if not exists testdb keep 3650 duration 1000 cachemodel {cache_value}") + tdSql.execute(" use testdb ") - tdSql.execute(f" create stable stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ + tdSql.execute(f" create stable testdb.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ uc2 bigint unsigned ,uc3 smallint unsigned , uc4 tinyint unsigned ) tags( t1 int , t2 bigint , t3 smallint , t4 tinyint , t5 float , t6 double , t7 bool , t8 binary(36)\ , t9 nchar(36) , t10 int unsigned , t11 bigint unsigned ,t12 smallint unsigned , t13 tinyint unsigned ,t14 timestamp ) ") tdSql.execute( ''' - create table t1 + create table testdb.t1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) ''' ) for i in range(4): tdSql.execute( - f'create table ct{i+1} using stb1 tags ( {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" ,{111*i}, {1*i},{1*i},{1*i},now())') + f'create table testdb.ct{i+1} using stb1 tags ( {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" ,{111*i}, {1*i},{1*i},{1*i},now())') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i} )" + f"insert into testdb.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i} )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i})" + f"insert into testdb.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i})" ) tdSql.execute( - "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a ,0,0,0,0)") + "insert into testdb.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a ,0,0,0,0)") tdSql.execute( - "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a , 999 , 9999 , 9 , 9)") + "insert into testdb.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a , 999 , 9999 , 9 , 9)") tdSql.execute( - "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a , 999 , 99999 , 9 , 9)") + "insert into testdb.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a , 999 , 99999 , 9 , 9)") tdSql.execute( - "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a ,999 , 99999 , 9 , 9)") + "insert into testdb.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a ,999 , 99999 , 9 , 9)") tdSql.execute( - "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") + "insert into testdb.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") tdSql.execute( - "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") + "insert into testdb.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") tdSql.execute( - "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL ) ") + "insert into testdb.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL ) ") tdSql.execute( - f'''insert into t1 values + f'''insert into testdb.t1 values ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) @@ -215,37 +220,25 @@ class TDTestCase: def test_errors(self): tdSql.execute("use testdb") - # bug need fix - # tdSql.query("select last_row(c1 ,NULL) from t1") + # bug need fix + tdSql.query("select last_row(c1 ,NULL) from testdb.t1") error_sql_lists = [ - "select last_row from t1", - "select last_row(-+--+c1) from t1", - # "select +-last_row(c1) from t1", - # "select ++-last_row(c1) from t1", - # "select ++--last_row(c1) from t1", - # "select - -last_row(c1)*0 from t1", - # "select last_row(tbname+1) from t1 ", - "select last_row(123--123)==1 from t1", - "select last_row(c1) as 'd1' from t1", - "select last_row(c1 ,NULL) from t1", - "select last_row(,) from t1;", - "select last_row(abs(c1) ab from t1)", - "select last_row(c1) as int from t1", - "select last_row from stb1", - # "select last_row(-+--+c1) from stb1", - # "select +-last_row(c1) from stb1", - # "select ++-last_row(c1) from stb1", - # "select ++--last_row(c1) from stb1", - # "select - -last_row(c1)*0 from stb1", - # "select last_row(tbname+1) from stb1 ", - "select last_row(123--123)==1 from stb1", - "select last_row(c1) as 'd1' from stb1", - # "select last_row(c1 ,c2 ) from stb1", - "select last_row(c1 ,NULL) from stb1", - "select last_row(,) from stb1;", - "select last_row(abs(c1) ab from stb1)", - "select last_row(c1) as int from stb1" + "select last_row from testdb.t1", + "select last_row(-+--+c1) from testdb.t1", + "select last_row(123--123)==1 from testdb.t1", + "select last_row(c1) as 'd1' from testdb.t1", + #"select last_row(c1 ,NULL) from testdb.t1", + "select last_row(,) from testdb.t1;", + "select last_row(abs(c1) ab from testdb.t1)", + "select last_row(c1) as int from testdb.t1", + "select last_row from testdb.stb1", + "select last_row(123--123)==1 from testdb.stb1", + "select last_row(c1) as 'd1' from testdb.stb1", + #"select last_row(c1 ,NULL) from testdb.stb1", + "select last_row(,) from testdb.stb1;", + "select last_row(abs(c1) ab from testdb.stb1)", + "select last_row(c1) as int from testdb.stb1" ] for error_sql in error_sql_lists: tdSql.error(error_sql) @@ -253,7 +246,7 @@ class TDTestCase: def support_types(self): tdSql.execute("use testdb") tbnames = ["stb1", "t1", "ct1", "ct2"] - + for tbname in tbnames: tdSql.query("desc {}".format(tbname)) coltypes = tdSql.queryResult @@ -261,239 +254,241 @@ class TDTestCase: colname = coltype[0] col_note = coltype[-1] if col_note != "TAG": - abs_sql = "select last_row({}) from {}".format(colname, tbname) + abs_sql = "select last_row({}) from {}".format(colname, "testdb."+tbname) tdSql.query(abs_sql) - + def basic_abs_function(self): # basic query - tdSql.query("select c1 from ct3") + tdSql.query("select c1 from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select c1 from t1") + tdSql.query("select c1 from testdb.t1") tdSql.checkRows(12) - tdSql.query("select c1 from stb1") + tdSql.query("select c1 from testdb.stb1") tdSql.checkRows(25) # used for empty table , ct3 is empty - tdSql.query("select last_row(c1) from ct3") + tdSql.query("select last_row(c1) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c2) from ct3") + tdSql.query("select last_row(c2) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c3) from ct3") + tdSql.query("select last_row(c3) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c4) from ct3") + tdSql.query("select last_row(c4) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c5) from ct3") + tdSql.query("select last_row(c5) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c6) from ct3") + tdSql.query("select last_row(c6) from testdb.ct3") # used for regular table - # bug need fix - tdSql.query("select last_row(c1) from t1") + # bug need fix + tdSql.query("select last_row(c1) from testdb.t1") tdSql.checkData(0, 0, None) - tdSql.query("select last_row(c1) from ct4") + tdSql.query("select last_row(c1) from testdb.ct4") tdSql.checkData(0, 0, None) - tdSql.query("select last_row(c1) from stb1") + tdSql.query("select last_row(c1) from testdb.stb1") tdSql.checkData(0, 0, None) - # # bug need fix - tdSql.query("select last_row(c1), c2, c3 , c4, c5 from t1") + # # bug need fix + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.t1") tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) tdSql.checkData(0, 2, None) - # # bug need fix - tdSql.query("select last_row(c1), c2, c3 , c4, c5 from ct1") + # # bug need fix + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.ct1") tdSql.checkData(0, 0, 9) tdSql.checkData(0, 1, -99999) tdSql.checkData(0, 2, -999) tdSql.checkData(0, 3, None) tdSql.checkData(0, 4,-9.99000) - - # bug need fix - # tdSql.query("select last_row(c1), c2, c3 , c4, c5 from stb1 where tbname='ct1'") - # tdSql.checkData(0, 0, 9) - # tdSql.checkData(0, 1, -99999) - # tdSql.checkData(0, 2, -999) - # tdSql.checkData(0, 3, None) - # tdSql.checkData(0, 4,-9.99000) - - # bug fix - tdSql.query("select last_row(abs(c1)) from ct1") + + # bug need fix + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.stb1 where tbname='ct1'") + tdSql.checkData(0, 0, 9) + tdSql.checkData(0, 1, -99999) + tdSql.checkData(0, 2, -999) + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4,-9.99000) + + # bug fix + tdSql.query("select last_row(abs(c1)) from testdb.ct1") tdSql.checkData(0,0,9) - # # bug fix - tdSql.query("select last_row(c1+1) from ct1") - tdSql.query("select last_row(c1+1) from stb1") - tdSql.query("select last_row(c1+1) from t1") + # # bug fix + tdSql.query("select last_row(c1+1) from testdb.ct1") + tdSql.query("select last_row(c1+1) from testdb.stb1") + tdSql.query("select last_row(c1+1) from testdb.t1") # used for stable table - tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from ct1") + tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from testdb.ct1") tdSql.checkData(0,0,9) tdSql.checkData(0,1,-99999) tdSql.checkData(0,2,-999) tdSql.checkData(0,3,None) - # bug need fix - tdSql.query("select last_row(c1 ,c2 ,c3) from stb1 ") + # bug need fix + tdSql.query("select last_row(c1 ,c2 ,c3) from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,None) - # tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from stb1 where ts 5 ") + "select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,last_row(log(c1,2)-0.5) from db.ct4 where c1>5 ") tdSql.checkData(0, 0, 6) tdSql.checkData(0, 1, 6.000000000) tdSql.checkData(0, 2, 6.000000000) @@ -501,19 +496,19 @@ class TDTestCase: tdSql.checkData(0, 4, 2.084962501) tdSql.query( - "select last_row(c1,c2,c1+5) from ct4 where c1=5 ") + "select last_row(c1,c2,c1+5) from db.ct4 where c1=5 ") tdSql.checkData(0, 0, 5) tdSql.checkData(0, 1, 55555) tdSql.checkData(0, 2, 10.000000000) tdSql.query( - "select last(c1,c2,c1+5) from ct4 where c1=5 ") + "select last(c1,c2,c1+5) from db.ct4 where c1=5 ") tdSql.checkData(0, 0, 5) tdSql.checkData(0, 1, 55555) tdSql.checkData(0, 2, 10.000000000) tdSql.query( - "select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + "select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from db.ct4 where c1>log(c1,2) limit 1 ") tdSql.checkRows(1) tdSql.checkData(0, 0, 8) tdSql.checkData(0, 1, 88888) @@ -528,39 +523,40 @@ class TDTestCase: def check_boundary_values(self): tdSql.execute("drop database if exists bound_test") - tdSql.execute("create database if not exists bound_test cachemodel 'last_value'") + tdSql.execute("create database if not exists bound_test cachemodel 'LAST_ROW' ") + time.sleep(3) tdSql.execute("use bound_test") tdSql.execute( - "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + "create table bound_test.stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" ) - tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute(f'create table bound_test.sub1_bound using stb_bound tags ( 1 )') tdSql.execute( - f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) # check basic elem for table per row tdSql.query( - "select last(c1) ,last_row(c2), last_row(c3)+1 , last(c4)+1 from sub1_bound ") + "select last(c1) ,last_row(c2), last_row(c3)+1 , last(c4)+1 from bound_test.sub1_bound ") tdSql.checkData(0, 0, -2147483646) tdSql.checkData(0, 1, -9223372036854775806) tdSql.checkData(0, 2, -32765.000000000) tdSql.checkData(0, 3, -125.000000000) # check + - * / in functions tdSql.query( - "select last_row(c1+1) ,last_row(c2) , last(c3*1) , last(c4/2) from sub1_bound ") + "select last_row(c1+1) ,last_row(c2) , last(c3*1) , last(c4/2) from bound_test.sub1_bound ") def test_tag_compute_for_scalar_function(self): @@ -568,199 +564,227 @@ class TDTestCase: # bug need fix - tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") - tdSql.query("select c1 ,t1 from stb1 where t1 =0 ") + tdSql.query(" select sum(c1) from testdb.stb1 where t1+10 >1; ") + tdSql.query("select c1 ,t1 from testdb.stb1 where t1 =0 ") tdSql.checkRows(13) - tdSql.query("select last_row(c1,t1) from stb1 ") + tdSql.query("select last_row(c1,t1) from testdb.stb1 ") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,3) + tdSql.query("select last_row(c1),t1 from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,3) - # tdSql.query("select last_row(c1),t1 from stb1 ") - # tdSql.checkData(0,0,None) - # tdSql.checkData(0,1,3) - tdSql.query("select last_row(c1,t1),last(t1) from stb1 ") + tdSql.query("select last_row(c1,t1),last(t1) from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,3) tdSql.checkData(0,2,3) - tdSql.query("select last_row(t1) from stb1 where t1 >0 ") + tdSql.query("select last_row(t1) from testdb.stb1 where t1 >0 ") tdSql.checkRows(1) tdSql.checkData(0,0,3) - tdSql.query("select last_row(t1) from stb1 where t1 =3 ") + tdSql.query("select last_row(t1) from testdb.stb1 where t1 =3 ") tdSql.checkRows(1) tdSql.checkData(0,0,3) - tdSql.query("select last_row(t1) from stb1 where t1 =2") + tdSql.query("select last_row(t1) from testdb.stb1 where t1 =2") tdSql.checkRows(0) # nest query for last_row - # tdSql.query("select last_row(t1) from (select c1 ,t1 from stb1)") - # tdSql.checkData(0,0,61) - # tdSql.query("select distinct(c1) ,t1 from stb1") - # tdSql.checkRows(20) - tdSql.query("select last_row(c1) from (select _rowts , c1 ,t1 from stb1)") + tdSql.query("select last_row(t1) from (select ts , c1 ,t1 from testdb.stb1)") + tdSql.checkData(0,0,3) + tdSql.query("select distinct(c1) ,t1 from testdb.stb1") + tdSql.checkRows(20) + tdSql.query("select last_row(c1) from (select _rowts , c1 ,t1 from testdb.stb1)") tdSql.checkData(0,0,None) - tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from testdb.stb1)") tdSql.checkData(0,0,None) - tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from stb1)") - tdSql.checkData(0,1,None,None) + tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from testdb.stb1)") + tdSql.checkData(0,1,None) - tdSql.query("select ts , last_row(c1) ,c1 from (select ts , max(c1) c1 ,t1 from stb1 where ts >now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts ="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s) fill(NULL)') - # tdSql.checkRows(40) - # tdSql.checkData(0,0,None) - tdSql.query('select max(c1) from stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') + tdSql.query("select last_row(c1) , ceil(t1) ,c2 from testdb.stb1 partition by t1 order by t1") + tdSql.checkRows(2) + + tdSql.query("select last_row(c1) , abs(t1) ,c2 from testdb.stb1 partition by abs(c1) order by abs(c1)") + tdSql.checkRows(11) + + tdSql.query("select abs(last_row(c1)) , abs(floor(t1)) ,floor(c2) from testdb.stb1 partition by abs(floor(c1)) order by abs(c1)") + tdSql.checkRows(11) + + tdSql.query("select last_row(ceil(c1-2)) , abs(floor(t1+1)) ,floor(c2-c1) from testdb.stb1 partition by abs(floor(c1)) order by abs(c1)") + tdSql.checkRows(11) + + # interval + tdSql.query("select last_row(c1) from testdb.stb1 interval(50s) sliding(30s)") + tdSql.checkRows(27) + + tdSql.query("select last_row(c1) from testdb.ct1 interval(50s) sliding(30s)") tdSql.checkRows(5) + last_row_result = tdSql.queryResult + tdSql.query("select last(c1) from testdb.ct1 interval(50s) sliding(30s)") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + # bug need fix + tdSql.query('select max(c1) from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" interval(50d) sliding(30d) fill(NULL)') + tdSql.checkRows(8) + tdSql.checkData(7,0,None) + tdSql.query('select last_row(c1) from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" interval(50d) sliding(30d) fill(value ,2 )') + tdSql.checkRows(8) + tdSql.checkData(7,0,2) - def support_super_table_test(self): - tdSql.execute(" use testdb ") - self.check_result_auto( " select c1 from stb1 order by ts " , "select abs(c1) from stb1 order by ts" ) - self.check_result_auto( " select c1 from stb1 order by tbname " , "select abs(c1) from stb1 order by tbname" ) - self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) - self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) - - self.check_result_auto( " select t1,c1 from stb1 order by ts " , "select t1, abs(c1) from stb1 order by ts" ) - self.check_result_auto( " select t2,c1 from stb1 order by tbname " , "select t2 ,abs(c1) from stb1 order by tbname" ) - self.check_result_auto( " select t3,c1 from stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from stb1 where c1 > 0 order by tbname" ) - self.check_result_auto( " select t4,c1 from stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from stb1 where c1 > 0 order by tbname" ) - pass + tdSql.query('select last_row(c1) from testdb.stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') + tdSql.query('select last_row(c1) from (select ts , c1 from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" ) interval(10s) sliding(5s)') + # join + tdSql.query("use test") + tdSql.query("select last(sub_tb_1.c1), last(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + tdSql.checkCols(2) + last_row_result = tdSql.queryResult + tdSql.query("select last_row(sub_tb_1.c1), last_row(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") - def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - # tdSql.prepare() + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) - tdLog.printNoPrefix("==========step1:create table ==============") + tdSql.query("select last(*), last(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + + last_row_result = tdSql.queryResult + tdSql.query("select last_row(*), last_row(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + + tdSql.query("select last(*), last_row(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + + tdSql.query("select last_row(*), last(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + + + def support_super_table_test(self): + tdSql.execute(" use testdb ") + self.check_result_auto( " select c1 from testdb.stb1 order by ts " , "select abs(c1) from testdb.stb1 order by ts" ) + self.check_result_auto( " select c1 from testdb.stb1 order by tbname " , "select abs(c1) from testdb.stb1 order by tbname" ) + self.check_result_auto( " select c1 from testdb.stb1 where c1 > 0 order by tbname " , "select abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c1 from testdb.stb1 where c1 > 0 order by tbname " , "select abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c1 from testdb.stb1 order by ts " , "select t1, abs(c1) from testdb.stb1 order by ts" ) + self.check_result_auto( " select t2,c1 from testdb.stb1 order by tbname " , "select t2 ,abs(c1) from testdb.stb1 order by tbname" ) + self.check_result_auto( " select t3,c1 from testdb.stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t4,c1 from testdb.stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) + pass - self.prepare_datas() - self.prepare_tag_datas() + def basic_query(self): tdLog.printNoPrefix("==========step2:test errors ==============") @@ -788,12 +812,41 @@ class TDTestCase: tdLog.printNoPrefix("==========step7: check result of query ============") - self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step) tdLog.printNoPrefix("==========step8: check abs result of stable query ============") self.support_super_table_test() + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + # tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + # cache_last 0 + self.prepare_datas("'NONE' ") + self.prepare_tag_datas("'NONE'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'NONE'") + self.basic_query() + + # cache_last 1 + self.prepare_datas("'LAST_ROW'") + self.prepare_tag_datas("'LAST_ROW'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_ROW'") + self.basic_query() + + # cache_last 2 + self.prepare_datas("'LAST_VALUE'") + self.prepare_tag_datas("'LAST_VALUE'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_VALUE'") + self.basic_query() + + # cache_last 3 + self.prepare_datas("'BOTH'") + self.prepare_tag_datas("'BOTH'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'BOTH'") + self.basic_query() + + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/7-tmq/stbTagFilter.py b/tests/system-test/7-tmq/stbTagFilter.py index 65609629bc92f2e662d9abc9c799ccdf2ca2e7ae..1bb3d24bdea0eb4c3510ea0e3cc4aeb27389d726 100644 --- a/tests/system-test/7-tmq/stbTagFilter.py +++ b/tests/system-test/7-tmq/stbTagFilter.py @@ -5,103 +5,250 @@ import time import socket import os import threading +from enum import Enum from util.log import * from util.sql import * from util.cases import * from util.dnodes import * -from util.common import * sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: + def __init__(self): + self.snapshot = 0 + self.vgroups = 4 + self.ctbNum = 1 + self.rowsPerTbl = 10000 + def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) - #tdSql.init(conn.cursor(), logSql) # output sql.txt file + tdSql.init(conn.cursor(), False) - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'db2', + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', - 'vgroups': 1, + 'vgroups': 4, 'stbName': 'stb', 'colPrefix': 'c', 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 10, + 'ctbNum': 1, + 'rowsPerTbl': 100000, + 'batchNum': 1200, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 10, + 'pollDelay': 3, 'showMsg': 1, - 'showRow': 1} + 'showRow': 1, + 'snapshot': 0} - topicNameList = ['topic1'] - expectRowsList = [] + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=1,replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) tdLog.info("create stb") - tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("insert data") - tmqCom.asyncInsertData(paraDict) + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 100000, + 'batchNum': 3000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + # update to half tables + # paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) + # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + consumerId = 0 + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromStb1 + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("create topics from stb with filter") - # queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) - queryString = "select * from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdLog.info("run select sql from db") + tdSql.query(queryString) + expectrowcnt = tdSql.getRows() + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + if totalConsumeRows != expectrowcnt: + tdLog.exit("tmq consume rows error!") + + tmqCom.checkFileContent(consumerId, queryString) + + tdSql.query("drop topic %s"%topicFromStb1) + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 5000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("restart taosd to ensure that the data falls into the disk") + tdSql.query("flush database %s"%(paraDict['dbName'])) + + # update to half tables + paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl / 2) + paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) + tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tmqCom.initConsumerTable() + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # start tmq consume processor - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 - topicList = topicNameList[0] - ifcheckdata = 0 + tdSql.execute(sqlString) + + # paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + consumerId = 1 + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromStb1 + ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:2000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) - - # tmqCom.getStartCommitNotifyFromTmqsim() - tmqCom.getStartConsumeNotifyFromTmqsim() - tdLog.info("create some new ctb") - paraDict['ctbStartIdx'] = paraDict['ctbStartIdx'] + paraDict['ctbNum'] - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data into new ctb") - pThread = tmqCom.asyncInsertData(paraDict) - - pThread.join() - tdLog.info("wait insert end") - tdSql.query(queryString) - expectRowsList.append(tdSql.getRows()) - - tdLog.info("wait the consume result") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tdLog.info("insert process end, and start to check consume result") expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() - if expectRowsList[0] != resultList[0]: - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) - tdLog.exit("0 tmq consume rows error!") + tdLog.info("act consume rows: %d, act insert rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsInserted, expectrowcnt)) - time.sleep(10) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) + if totalConsumeRows != expectrowcnt: + tdLog.exit("tmq consume rows error!") + + # tmqCom.checkFileContent(consumerId, queryString) - tdLog.printNoPrefix("======== test case 1 end ...... ") + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 2 end ...... ") def run(self): tdSql.prepare() + self.prepareTestEnv() + tdLog.printNoPrefix("=============================================") + tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") self.tmqCase1() + # self.tmqCase2() + + # self.prepareTestEnv() + # tdLog.printNoPrefix("====================================================================") + # tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + # self.snapshot = 1 + # self.tmqCase1() + # self.tmqCase2() + def stop(self): tdSql.close() diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 9eb8d531f796ace5fdbc6e7d3708253ef2bdf06e..28a1a1fd4ab64d8f7d3b6cd738df044e098075a5 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -320,7 +320,7 @@ class TDTestCase: tdSql.prepare() self.tmqCase1() - # self.tmqCase2() + self.tmqCase2() self.tmqCase3() def stop(self): diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ddbbefe739642f2a89666a171800e8cc823a460d..0b22aebb0d6c46e960f7b31196452df876cf98c6 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -10,7 +10,7 @@ python3 ./test.py -f 0-others/taosdMonitor.py python3 ./test.py -f 0-others/udfTest.py python3 ./test.py -f 0-others/udf_create.py python3 ./test.py -f 0-others/udf_restart_taosd.py -python3 ./test.py -f 0-others/cachelast.py +python3 ./test.py -f 0-others/cachemodel.py python3 ./test.py -f 0-others/udf_cfg1.py python3 ./test.py -f 0-others/udf_cfg2.py @@ -32,7 +32,10 @@ python3 ./test.py -f 1-insert/block_wise.py python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py +python3 ./test.py -f 1-insert/update_data.py + python3 ./test.py -f 2-query/db.py + python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py @@ -126,7 +129,7 @@ python3 ./test.py -f 2-query/count_partition.py python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 2-query/max_partition.py - +python3 ./test.py -f 2-query/last_row.py python3 ./test.py -f 6-cluster/5dnode1mnode.py #BUG python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 @@ -274,6 +277,7 @@ python3 ./test.py -f 2-query/irate.py -Q 2 python3 ./test.py -f 2-query/function_null.py -Q 2 python3 ./test.py -f 2-query/count_partition.py -Q 2 python3 ./test.py -f 2-query/max_partition.py -Q 2 +python3 ./test.py -f 2-query/last_row.py -Q 2 #------------querPolicy 3----------- @@ -361,3 +365,4 @@ python3 ./test.py -f 2-query/irate.py -Q 3 python3 ./test.py -f 2-query/function_null.py -Q 3 python3 ./test.py -f 2-query/count_partition.py -Q 3 python3 ./test.py -f 2-query/max_partition.py -Q 3 +python3 ./test.py -f 2-query/last_row.py -Q 3 diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 61c50fb0e8e37effe7a89c1470e09c0e21fa14ce..784b45c92bcb2b1b8e30cfdd7d60b31f279e2e8d 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -596,7 +596,8 @@ void printParaIntoFile() { g_fp = pFile; time_t tTime = taosGetTimestampSec(); - struct tm tm = *taosLocalTime(&tTime, NULL); + struct tm tm; + taosLocalTime(&tTime, &tm); taosFprintfFile(pFile, "###################################################################\n"); taosFprintfFile(pFile, "# configDir: %s\n", configDir); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 5459e3f159ba2d500652c2628c1c57ac0241eb89..7030f6fea6153739f05f0d3a7966b61cc9b778ad 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -171,7 +171,8 @@ static void printHelp() { char* getCurrentTimeString(char* timeString) { time_t tTime = taosGetTimestampSec(); - struct tm tm = *taosLocalTime(&tTime, NULL); + struct tm tm; + taosLocalTime(&tTime, &tm); sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); @@ -420,18 +421,6 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { ms = val % 1000; } - /* - comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -443,8 +432,9 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { } } - struct tm* ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm ptm; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index aaad76bb530cbb4f4ae1fa6f401f08db56adedbc..b993a8dbf1377c57af81d767173f5bfd27b688e3 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -635,7 +635,7 @@ bool simCreateTaosdConnect(SScript *script, char *rest) { bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { char timeStr[30] = {0}; time_t tt; - struct tm *tp; + struct tm tp; SCmdLine *line = &script->lines[script->linePos]; int32_t ret = -1; @@ -768,20 +768,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } else { tt = (*(int64_t *)row[i]) / 1000000000; } - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif - tp = taosLocalTime(&tt, NULL); - strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", tp); + taosLocalTime(&tt, &tp); + strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp); if (precision == TSDB_TIME_PRECISION_MILLI) { sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000)); } else if (precision == TSDB_TIME_PRECISION_MICRO) { diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 5b6dda4dae008364ed11c352e6fd7c8adfa0fc76..b9f7610be83ce77c6e76d9821e9b77bd0816284d 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -175,7 +175,7 @@ SScript *simBuildScriptObj(char *fileName) { SScript *simParseScript(char *fileName) { TdFilePtr pFile; int32_t tokenLen, lineNum = 0; - char *buffer = NULL, name[128], *token, *rest; + char buffer[10*1024], name[128], *token, *rest; SCommand *pCmd; SScript *script; @@ -195,7 +195,7 @@ SScript *simParseScript(char *fileName) { simResetParser(); while (!taosEOFFile(pFile)) { - if (taosGetLineFile(pFile, &buffer) == -1) continue; + if (taosGetsFile(pFile, sizeof(buffer) - 1, buffer) == -1) continue; lineNum++; int32_t cmdlen = (int32_t)strlen(buffer); @@ -240,7 +240,6 @@ SScript *simParseScript(char *fileName) { return NULL; } } - if(buffer != NULL) taosMemoryFree(buffer); taosCloseFile(&pFile); script = simBuildScriptObj(fileName); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 4cfa46bd3cb57cc14741fdc5a907d11297ad66ae..14cab041f6c458ac36461f54745d7b4962c6b2e2 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -231,18 +231,6 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { ms = val % 1000; } - /* - comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -254,8 +242,9 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { } } - struct tm *ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm ptm = {0}; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); @@ -748,13 +737,6 @@ int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool ver void shellReadHistory() { SShellHistory *pHistory = &shell.history; - int64_t file_size; - if (taosStatFile(pHistory->file, &file_size, NULL) != 0) { - return; - } else if (file_size > SHELL_MAX_COMMAND_SIZE) { - taosRemoveFile(pHistory->file); - return; - } TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return; @@ -774,10 +756,29 @@ void shellReadHistory() { if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); + int64_t file_size; + if (taosStatFile(pHistory->file, &file_size, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) { + fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout); + TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC); + if (pFile == NULL) return; + int32_t endIndex = pHistory->hstart; + if (endIndex != 0) { + endIndex = pHistory->hend; + } + for (int32_t i = (pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; i != endIndex;) { + taosFprintfFile(pFile, "%s\n", pHistory->hist[i]); + i = (i + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; + } + taosFprintfFile(pFile, "%s\n", pHistory->hist[endIndex]); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + } + pHistory->hend = pHistory->hstart; } void shellWriteHistory() { SShellHistory *pHistory = &shell.history; + if (pHistory->hend == pHistory->hstart) return; TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND); if (pFile == NULL) return; diff --git a/tools/taos-tools b/tools/taos-tools index b7b922268c4a06d9db77ffdfde0726f3d9900b72..0b8a3373bb7548f8106d13e7d3b0a988d3c4d48a 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit b7b922268c4a06d9db77ffdfde0726f3d9900b72 +Subproject commit 0b8a3373bb7548f8106d13e7d3b0a988d3c4d48a