diff --git a/README.md b/README.md index 67ca560130199b4536b4803f30970dd4469d2c57..00d36efb5a630a44eeeaa9b3b47df5c649a0a305 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -English | [简体中文](README-CN.md) | [Learn more about TSDB](https://tdengine.com/tsdb/) +English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine.com) | [Learn more about TSDB](https://tdengine.com/tsdb/) # What is TDengine？ @@ -33,7 +33,7 @@ TDengine is an open source, high-performance, cloud native [time-series database - **[Open Source](https://tdengine.com/tdengine/open-source-time-series-database/)**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered 18.8k stars on GitHub. There is an active developer community, and over 139k running instances worldwide. -For a full list of TDengine competitive advantages, please [check here](https://tdengine.com/tdengine/) +For a full list of TDengine competitive advantages, please [check here](https://tdengine.com/tdengine/). The easiest way to experience TDengine is through [TDengine Cloud](https://cloud.tdengine.com). # Documentation diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 2dc7622f4601ad8b1fb5c578c6aa8b5f0df02daf..a1a6cd4519294fa99a5bee41ae21aad4c1731373 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -37,6 +37,11 @@ if(${BUILD_WITH_ICONV}) cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() +# jemalloc +if(${JEMALLOC_ENABLED}) + cat("${TD_SUPPORT_DIR}/jemalloc_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + # msvc regex if(${BUILD_MSVCREGEX}) cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -258,6 +263,19 @@ if(${BUILD_PTHREAD}) target_link_libraries(pthread INTERFACE libpthreadVC3) endif() +# jemalloc +if(${JEMALLOC_ENABLED}) + include(ExternalProject) + ExternalProject_Add(jemalloc + PREFIX "jemalloc" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls + BUILD_COMMAND ${MAKE} + ) + INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include) +endif() + # crashdump if(${BUILD_CRASHDUMP}) add_executable(dumper "crashdump/dumper/dumper.c") diff --git a/docs/en/05-get-started/01-docker.md b/docs/en/05-get-started/01-docker.md index 39fdfeeb064df8fe7bf1c58a5ea247859f275846..ac273daba48d39bd0777ffdb30f431c6521d13b3 100644 --- a/docs/en/05-get-started/01-docker.md +++ b/docs/en/05-get-started/01-docker.md @@ -5,13 +5,25 @@ title: Quick Install on Docker This document describes how to install TDengine in a Docker container and perform queries and inserts. -- The easiest way to explore TDengine is through [TDengine Cloud](http://cloud.tdengine.com). +- The easiest way to explore TDengine is through [TDengine Cloud](http://cloud.tdengine.com). - To get started with TDengine in a non-containerized environment, see [Quick Install from Package](../../get-started/package). - If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine). ## Run TDengine -If Docker is already installed on your computer, run the following command: +If Docker is already installed on your computer, pull the latest TDengine Docker container image: + +```shell +docker pull tdengine/tdengine:latest +``` + +Or the container image of specific version: + +```shell +docker pull tdengine/tdengine:3.0.1.4 +``` + +And then run the following command: ```shell docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 243ede5fcbe157c933b655fe91e95d1d87084a7e..2b39c4b9e529e7dd3d9acd8b1d9107cf02038063 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -864,7 +864,7 @@ INTERP(expr) - `INTERP` is used to get the value that matches the specified time slice from a column. If no such value exists an interpolation value will be returned based on `FILL` parameter. - The input data of `INTERP` is the value of the specified column and a `where` clause can be used to filter the original data. If no `where` condition is specified then all original data is the input. - The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1<=timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified. -- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. +- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. The parameter `EVERY` must be an integer, with no quotes, with a time unit of: b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds. - Interpolation is performed based on `FILL` parameter. - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. - Pseudo column `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4). diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index 8f63dddfb711a98840cb423d1a4b0c1556a7b5fd..bde5e3303491ee0b748be2372d7fa18b94f6b971 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -23,7 +23,7 @@ There are two ways to install taosBenchmark: TaosBenchmark needs to be executed on the terminal of the operating system, it supports two configuration methods: [Command-line arguments](#command-line-arguments-in-detail) and [JSON configuration file](#configuration-file-parameters-in-detail). These two methods are mutually exclusive. Users can use `-f ` to specify a configuration file. When running taosBenchmark with command-line arguments to control its behavior, users should use other parameters for configuration, but not the `-f` parameter. In addition, taosBenchmark offers a special way of running without parameters. -taosBenchmark supports the complete performance testing of TDengine by providing functionally to write, query, and subscribe. These three functions are mutually exclusive, users can only select one of them each time taosBenchmark runs. The query and subscribe functionalities are only configurable using a json configuration file by specifying the parameter `filetype`, while write can be performed through both the command-line and a configuration file. If you want to test the performance of queries or data subscriptionm configure taosBenchmark with the configuration file. You can modify the value of the `filetype` parameter to specify the function that you want to test. +taosBenchmark supports the complete performance testing of TDengine by providing functionally to write, query, and subscribe. These three functions are mutually exclusive, users can only select one of them each time taosBenchmark runs. The query and subscribe functionalities are only configurable using a json configuration file by specifying the parameter `filetype`, while write can be performed through both the command-line and a configuration file. If you want to test the performance of queries or data subscription configure taosBenchmark with the configuration file. You can modify the value of the `filetype` parameter to specify the function that you want to test. **Make sure that the TDengine cluster is running correctly before running taosBenchmark. ** @@ -340,7 +340,7 @@ The configuration parameters for specifying super table tag columns and data col - **values**: The value field of the nchar/binary column/label, which will be chosen randomly from the values. -- **sma**: Insert the column into the BSMA. Enter `yes` or `no`. The default is `no`. +- **sma**: Insert the column into the SMA. Enter `yes` or `no`. The default is `no`. #### insertion behavior configuration parameters diff --git a/docs/examples/c/tmq_example.c b/docs/examples/c/tmq_example.c index 19adaad116ef65673f5541b5216ce12d2d9151c7..a3bade308a4ce41f556878d90861e299893315c4 100644 --- a/docs/examples/c/tmq_example.c +++ b/docs/examples/c/tmq_example.c @@ -184,22 +184,54 @@ void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { tmq_t* build_consumer() { tmq_conf_res_t code; tmq_conf_t* conf = tmq_conf_new(); + code = tmq_conf_set(conf, "enable.auto.commit", "true"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + tmq_conf_destroy(conf); + return NULL; + } + code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + tmq_conf_destroy(conf); + return NULL; + } + code = tmq_conf_set(conf, "group.id", "cgrpName"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + tmq_conf_destroy(conf); + return NULL; + } + code = tmq_conf_set(conf, "client.id", "user defined name"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + tmq_conf_destroy(conf); + return NULL; + } + code = tmq_conf_set(conf, "td.connect.user", "root"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + tmq_conf_destroy(conf); + return NULL; + } + code = tmq_conf_set(conf, "td.connect.pass", "taosdata"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + tmq_conf_destroy(conf); + return NULL; + } + code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + tmq_conf_destroy(conf); + return NULL; + } + code = tmq_conf_set(conf, "experimental.snapshot.enable", "false"); - if (TMQ_CONF_OK != code) return NULL; + if (TMQ_CONF_OK != code) { + tmq_conf_destroy(conf); + return NULL; + } tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index c65e00384d81eed452ef48a2efcf92d671868b1d..e772447db087c119fae5571b574b419950437b9d 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -8,7 +8,19 @@ description: 使用 Docker 快速体验 TDengine 的高效写入和查询 ## 启动 TDengine -如果已经安装了 Docker，只需执行下面的命令： +如果已经安装了 Docker，首先拉取最新的 TDengine 容器镜像： + +```shell +docker pull tdengine/tdengine:latest +``` + +或者指定版本的容器镜像： + +```shell +docker pull tdengine/tdengine:3.0.1.4 +``` + +然后只需执行下面的命令： ```shell docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 0d6aad62401daf76737caf803461c187189cb76f..6a6d9e3878638357131985757418c3c2aa5cd8ef 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -340,7 +340,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **values** : nchar/binary 列/标签的值域，将从值中随机选择。 -- **sma**: 将该列加入bsma中，值为 "yes" 或者 "no"，默认为 "no"。 +- **sma**: 将该列加入 SMA 中，值为 "yes" 或者 "no"，默认为 "no"。 #### 插入行为配置参数 diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 6db2ffb7ea381367e8592259354c8b91f182b678..0e22985059907956c207d7c40103480d0913e65e 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -103,6 +103,7 @@ extern bool tsKeepColumnName; // client extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; +extern int32_t tsMaxMemUsedByInsert; // build info extern char version[]; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index ee566d759a980cf42cc8a77e81a7cc2e639b707c..c3caac00ad62a030babfad2843e7e3eb47884dc3 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -303,6 +303,10 @@ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t re int32_t catalogClearCache(void); +SMetaData* catalogCloneMetaData(SMetaData* pData); + +void catalogFreeMetaData(SMetaData* pData); + /** * Destroy catalog and relase all resources */ diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5ee097bd9254970536c61caace0fdde3f6a11d9d..e1acf0dd6ac74954c3be320e9792b7a78206a4bf 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -385,7 +385,6 @@ typedef struct SCmdMsgInfo { SEpSet epSet; void* pMsg; int32_t msgLen; - void* pExtension; // todo remove it soon } SCmdMsgInfo; typedef enum EQueryExecMode { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index b1a937910dfe8defd107ec525afd20edfc639aaf..bcd2316baf22f896916bb0eb5b3f6ae654afb48a 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -33,6 +33,13 @@ typedef struct SStmtCallback { int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**); } SStmtCallback; +typedef struct SParseCsvCxt { + TdFilePtr fp; // last parsed file + int32_t tableNo; // last parsed table + SName tableName; // last parsed table + const char* pLastSqlPos; // the location of the last parsed sql +} SParseCsvCxt; + typedef struct SParseContext { uint64_t requestId; int64_t requestRid; @@ -57,6 +64,8 @@ typedef struct SParseContext { SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableVgroupPos; // sql table pos => catalog data pos int64_t allocatorId; + bool needMultiParse; + SParseCsvCxt csvCxt; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); @@ -67,6 +76,8 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, SQuery* pQuery); +void qDestroyParseContext(SParseContext* pCxt); + void qDestroyQuery(SQuery* pQueryNode); int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); diff --git a/include/os/os.h b/include/os/os.h index 71966061a19a175d816010ff6425b4004b1f2223..d646ffe4fdc4a3118c0594b406031a6287256c54 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -51,6 +51,9 @@ extern "C" { #endif #else +#ifndef __func__ +#define __func__ __FUNCTION__ +#endif #include #include #ifndef TD_USE_WINSOCK diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 535a436c6c17dfd90603f40dc189bcff5b2902d2..f815adfeaa8e77473acd67a01f945aa360e1d5e7 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -380,16 +380,25 @@ void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); // --- mq void hbMgrInitMqHbRspHandle(); +typedef struct SSqlCallbackWrapper { + SParseContext* pParseCtx; + SCatalogReq* pCatalogReq; + SMetaData* pResultMeta; + SRequestObj* pRequest; +} SSqlCallbackWrapper; + SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res); int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); -void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta); -int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); -int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList); -void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta); -int32_t removeMeta(STscObj* pTscObj, SArray* tbList); -int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog); -int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog); -bool qnodeRequired(SRequestObj* pRequest); +void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, SSqlCallbackWrapper* pWrapper); +int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); +int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList); +void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta); +int32_t removeMeta(STscObj* pTscObj, SArray* tbList); +int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog); +int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog); +bool qnodeRequired(SRequestObj* pRequest); +int32_t continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest); +void destorySqlCallbackWrapper(SSqlCallbackWrapper* pWrapper); #ifdef __cplusplus } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ec5c39062e21d9fda0a43426efa71d68930f3f54..7b437e47699ca623da62ceb496ab2a7337df7412 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -870,8 +870,9 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { // todo refacto the error code mgmt void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { - SRequestObj* pRequest = (SRequestObj*)param; - STscObj* pTscObj = pRequest->pTscObj; + SSqlCallbackWrapper* pWrapper = param; + SRequestObj* pRequest = pWrapper->pRequest; + STscObj* pTscObj = pRequest->pTscObj; pRequest->code = code; if (pResult) { @@ -882,7 +883,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { int32_t type = pRequest->type; if (TDMT_VND_SUBMIT == type || TDMT_VND_DELETE == type || TDMT_VND_CREATE_TABLE == type) { if (pResult) { - pRequest->body.resInfo.numOfRows = pResult->numOfRows; + pRequest->body.resInfo.numOfRows += pResult->numOfRows; // record the insert rows if (TDMT_VND_SUBMIT == type) { @@ -905,6 +906,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { schedulerFreeJob(&pRequest->body.queryJob, 0); qDestroyQuery(pRequest->pQuery); pRequest->pQuery = NULL; + destorySqlCallbackWrapper(pWrapper); doAsyncQuery(pRequest, true); return; } @@ -920,6 +922,15 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { pRequest->code = code1; } + if (pRequest->code == TSDB_CODE_SUCCESS && NULL != pWrapper->pParseCtx && pWrapper->pParseCtx->needMultiParse) { + code = continueInsertFromCsv(pWrapper, pRequest); + if (TSDB_CODE_SUCCESS == code) { + return; + } + } + + destorySqlCallbackWrapper(pWrapper); + // return to client pRequest->body.queryFp(pRequest->body.param, pRequest, code); } @@ -1020,76 +1031,86 @@ SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool vali return launchQueryImpl(pRequest, pQuery, false, NULL); } -void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta) { +static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, + SSqlCallbackWrapper* pWrapper) { + pRequest->type = pQuery->msgType; + + SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); + SPlanContext cxt = {.queryId = pRequest->requestId, + .acctId = pRequest->pTscObj->acctId, + .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite, + .pMsg = pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pUser = pRequest->pTscObj->user, + .sysInfo = pRequest->pTscObj->sysInfo, + .allocatorId = pRequest->allocatorRefId}; + SQueryPlan* pDag = NULL; + int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); + if (code) { + tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + } else { + pRequest->body.subplanNum = pDag->numOfSubplans; + } + + pRequest->metric.planEnd = taosGetTimestampUs(); + + if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { + SArray* pNodeList = NULL; + buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); + + SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self}; + SSchedulerReq req = { + .syncReq = false, + .localReq = (tsQueryPolicy == QUERY_POLICY_CLIENT), + .pConn = &conn, + .pNodeList = pNodeList, + .pDag = pDag, + .allocatorRefId = pRequest->allocatorRefId, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .execFp = schedulerExecCb, + .cbParam = pWrapper, + .chkKillFp = chkRequestKilled, + .chkKillParam = (void*)pRequest->self, + .pExecRes = NULL, + }; + code = schedulerExecJob(&req, &pRequest->body.queryJob); + taosArrayDestroy(pNodeList); + } else { + tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + destorySqlCallbackWrapper(pWrapper); + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } + + // todo not to be released here + taosArrayDestroy(pMnodeList); + + return code; +} + +void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, SSqlCallbackWrapper* pWrapper) { int32_t code = 0; pRequest->body.execMode = pQuery->execMode; + if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) { + destorySqlCallbackWrapper(pWrapper); + } switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: asyncExecLocalCmd(pRequest, pQuery); - return; + break; case QUERY_EXEC_MODE_RPC: code = asyncExecDdlQuery(pRequest, pQuery); break; case QUERY_EXEC_MODE_SCHEDULE: { - SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); - - pRequest->type = pQuery->msgType; - - SPlanContext cxt = {.queryId = pRequest->requestId, - .acctId = pRequest->pTscObj->acctId, - .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), - .pAstRoot = pQuery->pRoot, - .showRewrite = pQuery->showRewrite, - .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, - .pUser = pRequest->pTscObj->user, - .sysInfo = pRequest->pTscObj->sysInfo, - .allocatorId = pRequest->allocatorRefId}; - - SAppInstInfo* pAppInfo = getAppInfo(pRequest); - SQueryPlan* pDag = NULL; - code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); - if (code) { - tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), - pRequest->requestId); - } else { - pRequest->body.subplanNum = pDag->numOfSubplans; - } - - pRequest->metric.planEnd = taosGetTimestampUs(); - if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { - SArray* pNodeList = NULL; - buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); - - SRequestConnInfo conn = { - .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; - SSchedulerReq req = { - .syncReq = false, - .localReq = (tsQueryPolicy == QUERY_POLICY_CLIENT), - .pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .allocatorRefId = pRequest->allocatorRefId, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .execFp = schedulerExecCb, - .cbParam = pRequest, - .chkKillFp = chkRequestKilled, - .chkKillParam = (void*)pRequest->self, - .pExecRes = NULL, - }; - code = schedulerExecJob(&req, &pRequest->body.queryJob); - taosArrayDestroy(pNodeList); - } else { - tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), - pRequest->requestId); - pRequest->body.queryFp(pRequest->body.param, pRequest, code); - } - - // todo not to be released here - taosArrayDestroy(pMnodeList); + code = asyncExecSchQuery(pRequest, pQuery, pResultMeta, pWrapper); break; } case QUERY_EXEC_MODE_EMPTY_RESULT: diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 5b255a28ea8e125145f1bdb2a2bfcca5d6fc61f5..6f8cef7c0d50aff6dd76903691a0fc3521230d85 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -667,43 +667,47 @@ const char *taos_get_server_info(TAOS *taos) { return pTscObj->sDetailVer; } -typedef struct SqlParseWrapper { - SParseContext *pCtx; - SCatalogReq catalogReq; - SRequestObj *pRequest; -} SqlParseWrapper; - static void destoryTablesReq(void *p) { STablesReq *pRes = (STablesReq *)p; taosArrayDestroy(pRes->pTables); } -static void destorySqlParseWrapper(SqlParseWrapper *pWrapper) { - taosArrayDestroy(pWrapper->catalogReq.pDbVgroup); - taosArrayDestroy(pWrapper->catalogReq.pDbCfg); - taosArrayDestroy(pWrapper->catalogReq.pDbInfo); - taosArrayDestroyEx(pWrapper->catalogReq.pTableMeta, destoryTablesReq); - taosArrayDestroyEx(pWrapper->catalogReq.pTableHash, destoryTablesReq); - taosArrayDestroy(pWrapper->catalogReq.pUdf); - taosArrayDestroy(pWrapper->catalogReq.pIndex); - taosArrayDestroy(pWrapper->catalogReq.pUser); - taosArrayDestroy(pWrapper->catalogReq.pTableIndex); - taosArrayDestroy(pWrapper->pCtx->pTableMetaPos); - taosArrayDestroy(pWrapper->pCtx->pTableVgroupPos); - taosMemoryFree(pWrapper->pCtx); +static void destoryCatalogReq(SCatalogReq *pCatalogReq) { + if (NULL == pCatalogReq) { + return; + } + taosArrayDestroy(pCatalogReq->pDbVgroup); + taosArrayDestroy(pCatalogReq->pDbCfg); + taosArrayDestroy(pCatalogReq->pDbInfo); + taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq); + taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq); + taosArrayDestroy(pCatalogReq->pUdf); + taosArrayDestroy(pCatalogReq->pIndex); + taosArrayDestroy(pCatalogReq->pUser); + taosArrayDestroy(pCatalogReq->pTableIndex); + taosMemoryFree(pCatalogReq); +} + +void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) { + if (NULL == pWrapper) { + return; + } + destoryCatalogReq(pWrapper->pCatalogReq); + qDestroyParseContext(pWrapper->pParseCtx); + catalogFreeMetaData(pWrapper->pResultMeta); taosMemoryFree(pWrapper); } void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { - SqlParseWrapper *pWrapper = (SqlParseWrapper *)param; - SRequestObj *pRequest = pWrapper->pRequest; - SQuery *pQuery = pRequest->pQuery; + SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param; + SRequestObj *pRequest = pWrapper->pRequest; + SQuery *pQuery = pRequest->pQuery; pRequest->metric.ctgEnd = taosGetTimestampUs(); qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); if (code == TSDB_CODE_SUCCESS) { - code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); + code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); pRequest->stableQuery = pQuery->stableQuery; if (pQuery->pRoot) { pRequest->stmtType = pQuery->pRoot->type; @@ -712,6 +716,13 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { pRequest->metric.semanticEnd = taosGetTimestampUs(); + if (code == TSDB_CODE_SUCCESS && pWrapper->pParseCtx->needMultiParse) { + pWrapper->pResultMeta = catalogCloneMetaData(pResultMeta); + if (NULL == pWrapper->pResultMeta) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (code == TSDB_CODE_SUCCESS) { if (pQuery->haveResultSet) { setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols); @@ -722,15 +733,13 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { TSWAP(pRequest->tableList, (pQuery)->pTableList); TSWAP(pRequest->targetTableList, (pQuery)->pTargetTableList); - destorySqlParseWrapper(pWrapper); - - double el = (pRequest->metric.semanticEnd - pRequest->metric.ctgEnd)/1000.0; + double el = (pRequest->metric.semanticEnd - pRequest->metric.ctgEnd) / 1000.0; tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, elapsed time:%.2f ms, reqId:0x%" PRIx64, pRequest->self, el, pRequest->requestId); - launchAsyncQuery(pRequest, pQuery, pResultMeta); + launchAsyncQuery(pRequest, pQuery, pResultMeta, pWrapper); } else { - destorySqlParseWrapper(pWrapper); + destorySqlCallbackWrapper(pWrapper); qDestroyQuery(pRequest->pQuery); pRequest->pQuery = NULL; @@ -750,6 +759,16 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { } } +int32_t continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) { + qDestroyQuery(pRequest->pQuery); + pRequest->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pRequest->pQuery) { + return TSDB_CODE_OUT_OF_MEMORY; + } + retrieveMetaCallback(pWrapper->pResultMeta, pWrapper, TSDB_CODE_SUCCESS); + return TSDB_CODE_SUCCESS; +} + void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { int64_t connId = *(int64_t *)taos; taosAsyncQueryImpl(connId, sql, fp, param, false); @@ -786,37 +805,48 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { } void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { - SParseContext *pCxt = NULL; - STscObj *pTscObj = pRequest->pTscObj; - int32_t code = 0; + STscObj *pTscObj = pRequest->pTscObj; + SSqlCallbackWrapper *pWrapper = NULL; + int32_t code = TSDB_CODE_SUCCESS; if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) { code = pRequest->prevCode; - goto _error; } - code = createParseContext(pRequest, &pCxt); - if (code != TSDB_CODE_SUCCESS) { - goto _error; + if (TSDB_CODE_SUCCESS == code) { + pWrapper = taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); + if (pWrapper == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pWrapper->pRequest = pRequest; + } } - pCxt->mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCxt->pCatalog); - if (code != TSDB_CODE_SUCCESS) { - goto _error; + if (TSDB_CODE_SUCCESS == code) { + code = createParseContext(pRequest, &pWrapper->pParseCtx); } - pRequest->metric.syntaxStart = taosGetTimestampUs(); - - SCatalogReq catalogReq = {.forceUpdate = updateMetaForce, .qNodeRequired = qnodeRequired(pRequest)}; - code = qParseSqlSyntax(pCxt, &pRequest->pQuery, &catalogReq); - if (code != TSDB_CODE_SUCCESS) { - goto _error; + if (TSDB_CODE_SUCCESS == code) { + pWrapper->pParseCtx->mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pWrapper->pParseCtx->pCatalog); } - pRequest->metric.syntaxEnd = taosGetTimestampUs(); + if (TSDB_CODE_SUCCESS == code) { + pRequest->metric.syntaxStart = taosGetTimestampUs(); + + pWrapper->pCatalogReq = taosMemoryCalloc(1, sizeof(SCatalogReq)); + if (pWrapper->pCatalogReq == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pWrapper->pCatalogReq->forceUpdate = updateMetaForce; + pWrapper->pCatalogReq->qNodeRequired = qnodeRequired(pRequest); + code = qParseSqlSyntax(pWrapper->pParseCtx, &pRequest->pQuery, pWrapper->pCatalogReq); + } + + pRequest->metric.syntaxEnd = taosGetTimestampUs(); + } - if (!updateMetaForce) { + if (TSDB_CODE_SUCCESS == code && !updateMetaForce) { SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; if (NULL == pRequest->pQuery->pRoot) { atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); @@ -825,38 +855,26 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } } - SqlParseWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SqlParseWrapper)); - if (pWrapper == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; - } - - pWrapper->pCtx = pCxt; - pWrapper->pRequest = pRequest; - pWrapper->catalogReq = catalogReq; + if (TSDB_CODE_SUCCESS == code) { + SRequestConnInfo conn = {.pTrans = pWrapper->pParseCtx->pTransporter, + .requestId = pWrapper->pParseCtx->requestId, + .requestObjRefId = pWrapper->pParseCtx->requestRid, + .mgmtEps = pWrapper->pParseCtx->mgmtEpSet}; - SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, - .requestId = pCxt->requestId, - .requestObjRefId = pCxt->requestRid, - .mgmtEps = pCxt->mgmtEpSet}; + pRequest->metric.ctgStart = taosGetTimestampUs(); - pRequest->metric.ctgStart = taosGetTimestampUs(); - - code = catalogAsyncGetAllMeta(pCxt->pCatalog, &conn, &catalogReq, retrieveMetaCallback, pWrapper, - &pRequest->body.queryJob); - pCxt = NULL; - if (code == TSDB_CODE_SUCCESS) { - return; + code = catalogAsyncGetAllMeta(pWrapper->pParseCtx->pCatalog, &conn, pWrapper->pCatalogReq, retrieveMetaCallback, + pWrapper, &pRequest->body.queryJob); } -_error: - tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), - pRequest->requestId); - taosMemoryFree(pCxt); - - terrno = code; - pRequest->code = code; - pRequest->body.queryFp(pRequest->body.param, pRequest, code); + if (TSDB_CODE_SUCCESS != code) { + tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), + pRequest->requestId); + destorySqlCallbackWrapper(pWrapper); + terrno = code; + pRequest->code = code; + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } } static void fetchCallback(void *pResult, void *param, int32_t code) { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 195466061d2c19cbd6cedf6fadc08c8f2cc92bdc..c5633e73d0521c9304f1f23c778506cd9a0ad005 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -209,7 +209,7 @@ static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashOb } static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { - if(pBuf->buf){ + if (pBuf->buf) { memset(pBuf->buf, 0, pBuf->len); if (msg1) strncat(pBuf->buf, msg1, pBuf->len); int32_t left = pBuf->len - strlen(pBuf->buf); @@ -256,15 +256,15 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { while (result <= length) { result *= 2; } - if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){ + if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; } - if (type == TSDB_DATA_TYPE_NCHAR){ + if (type == TSDB_DATA_TYPE_NCHAR) { result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - }else if (type == TSDB_DATA_TYPE_BINARY){ + } else if (type == TSDB_DATA_TYPE_BINARY) { result = result + VARSTR_HEADER_SIZE; } return result; @@ -274,7 +274,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH ESchemaAction *action, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; for (int j = 0; j < taosArrayGetSize(cols); ++j) { - if(j == 0 && !isTag) continue; + if (j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); if (code != TSDB_CODE_SUCCESS) { @@ -286,12 +286,12 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - int32_t i = 0; - for ( ;i < length; i++) { + int32_t i = 0; + for (; i < length; i++) { taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); } - if (isTag){ + if (isTag) { i = 0; } else { i = 1; @@ -306,7 +306,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool return 0; } -static int32_t getBytes(uint8_t type, int32_t length){ +static int32_t getBytes(uint8_t type, int32_t length) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { return smlFindNearestPowerOf2(length, type); } else { @@ -314,21 +314,22 @@ static int32_t getBytes(uint8_t type, int32_t length){ } } -static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray* results, int32_t numOfCols, bool isTag) { +static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, + SArray *results, int32_t numOfCols, bool isTag) { for (int j = 0; j < taosArrayGetSize(cols); ++j) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); + SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); ESchemaAction action = SCHEMA_ACTION_NULL; smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); - if(action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG){ + if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); memcpy(field.name, kv->key, kv->keyLen); taosArrayPush(results, &field); - }else if(action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE){ + } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); - uint16_t newIndex = *index; - if(isTag) newIndex -= numOfCols; + uint16_t newIndex = *index; + if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); field->bytes = getBytes(kv->type, kv->length); } @@ -336,12 +337,11 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO return TSDB_CODE_SUCCESS; } -//static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, -// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ -static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns, SArray* pTags, - STableMeta *pTableMeta, ESchemaAction action){ - - SRequestObj* pRequest = NULL; +// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, +// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ +static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, + ESchemaAction action) { + SRequestObj *pRequest = NULL; SMCreateStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; SCmdMsgInfo pCmdMsg = {0}; @@ -363,24 +363,24 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns, goto end; } - if (action == SCHEMA_ACTION_CREATE_STABLE){ + if (action == SCHEMA_ACTION_CREATE_STABLE) { pReq.colVer = 1; pReq.tagVer = 1; pReq.suid = 0; pReq.source = TD_REQ_FROM_APP; - } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE){ + } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { pReq.colVer = pTableMeta->sversion; pReq.tagVer = pTableMeta->tversion + 1; pReq.suid = pTableMeta->uid; pReq.source = TD_REQ_FROM_TAOX; - } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE){ + } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { pReq.colVer = pTableMeta->sversion + 1; pReq.tagVer = pTableMeta->tversion; pReq.suid = pTableMeta->uid; pReq.source = TD_REQ_FROM_TAOX; } - if (pReq.numOfTags == 0){ + if (pReq.numOfTags == 0) { pReq.numOfTags = 1; SField field = {0}; field.type = TSDB_DATA_TYPE_NCHAR; @@ -412,7 +412,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray* pColumns, launchQueryImpl(pRequest, &pQuery, true, NULL); - if(pRequest->code == TSDB_CODE_SUCCESS){ + if (pRequest->code == TSDB_CODE_SUCCESS) { catalogRemoveTableMeta(info->pCatalog, pName); } code = pRequest->code; @@ -425,11 +425,11 @@ end: } static int32_t smlModifyDBSchemas(SSmlHandle *info) { - int32_t code = 0; - SHashObj *hashTmp = NULL; - STableMeta *pTableMeta = NULL; + int32_t code = 0; + SHashObj *hashTmp = NULL; + STableMeta *pTableMeta = NULL; - SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; strcpy(pName.dbname, info->pRequest->pDb); SRequestConnInfo conn = {0}; @@ -451,8 +451,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { - SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); - SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); @@ -463,8 +463,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } info->cost.numOfCreateSTables++; } else if (code == TSDB_CODE_SUCCESS) { - hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, + HASH_NO_LOCK); for (uint16_t i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); @@ -475,22 +475,25 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { if (code != TSDB_CODE_SUCCESS) { goto end; } - if (action != SCHEMA_ACTION_NULL){ - SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); + if (action != SCHEMA_ACTION_NULL) { + SArray *pColumns = + taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + SArray *pTags = + taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { SField field = {0}; field.type = pTableMeta->schema[i].type; field.bytes = pTableMeta->schema[i].bytes; strcpy(field.name, pTableMeta->schema[i].name); - if(i < pTableMeta->tableInfo.numOfColumns){ + if (i < pTableMeta->tableInfo.numOfColumns) { taosArrayPush(pColumns, &field); - }else{ + } else { taosArrayPush(pTags, &field); } } - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, pTableMeta->tableInfo.numOfColumns, true); + smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, + pTableMeta->tableInfo.numOfColumns, true); code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); if (code != TSDB_CODE_SUCCESS) { @@ -518,23 +521,26 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { if (code != TSDB_CODE_SUCCESS) { goto end; } - if (action != SCHEMA_ACTION_NULL){ - SArray* pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - SArray* pTags = taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); + if (action != SCHEMA_ACTION_NULL) { + SArray *pColumns = + taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + SArray *pTags = + taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { SField field = {0}; field.type = pTableMeta->schema[i].type; field.bytes = pTableMeta->schema[i].bytes; strcpy(field.name, pTableMeta->schema[i].name); - if(i < pTableMeta->tableInfo.numOfColumns){ + if (i < pTableMeta->tableInfo.numOfColumns) { taosArrayPush(pColumns, &field); - }else{ + } else { taosArrayPush(pTags, &field); } } - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, pTableMeta->tableInfo.numOfColumns, false); + smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, + pTableMeta->tableInfo.numOfColumns, false); code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); if (code != TSDB_CODE_SUCCESS) { @@ -847,7 +853,7 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { int64_t ts = 0; if (info->protocol == TSDB_SML_LINE_PROTOCOL) { -// uError("SML:data:%s,len:%d", data, len); + // uError("SML:data:%s,len:%d", data, len); ts = smlParseInfluxTime(info, data, len); } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { ts = smlParseOpenTsdbTime(info, data, len); @@ -877,7 +883,7 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { if (smlIsBinary(pVal->value, pVal->length)) { pVal->type = TSDB_DATA_TYPE_BINARY; pVal->length -= BINARY_ADD_LEN; - if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){ + if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } pVal->value += (BINARY_ADD_LEN - 1); @@ -887,7 +893,7 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { if (smlIsNchar(pVal->value, pVal->length)) { pVal->type = TSDB_DATA_TYPE_NCHAR; pVal->length -= NCHAR_ADD_LEN; - if(pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } pVal->value += (NCHAR_ADD_LEN - 1); @@ -1063,7 +1069,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab continue; } - if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } @@ -1224,7 +1230,7 @@ static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *c kv->value = value; kv->length = valueLen; if (isTag) { - if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } kv->type = TSDB_DATA_TYPE_NCHAR; @@ -1362,8 +1368,8 @@ static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) { static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { SHashObj *s1 = *(SHashObj **)key1; SHashObj *s2 = *(SHashObj **)key2; - SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); - SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); + SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); + SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); if (kv1->i < kv2->i) { @@ -1375,12 +1381,12 @@ static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { } } -static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){ - if(dataFormat){ +static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *cols) { + if (dataFormat) { void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT); - if(p == NULL){ + if (p == NULL) { taosArrayPush(oneTable->cols, &cols); - }else{ + } else { taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); } return TSDB_CODE_SUCCESS; @@ -1397,9 +1403,9 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col } void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT); - if(p == NULL){ + if (p == NULL) { taosArrayPush(oneTable->cols, &kvHash); - }else{ + } else { taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); } return TSDB_CODE_SUCCESS; @@ -1488,15 +1494,15 @@ static void smlDestroyInfo(SSmlHandle *info) { taosMemoryFreeClear(info); } -static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLProtocolType protocol, int8_t precision){ - int32_t code = TSDB_CODE_SUCCESS; - SSmlHandle* info = (SSmlHandle*)taosMemoryCalloc(1, sizeof(SSmlHandle)); +static SSmlHandle *smlBuildSmlInfo(STscObj *pTscObj, SRequestObj *request, SMLProtocolType protocol, int8_t precision) { + int32_t code = TSDB_CODE_SUCCESS; + SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { return NULL; } info->id = smlGenId(); - info->pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + info->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY); if (NULL == info->pQuery) { uError("SML:0x%" PRIx64 " create info->pQuery error", info->id); goto cleanup; @@ -1511,8 +1517,8 @@ static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLPr } ((SVnodeModifOpStmt *)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; - if (pTscObj){ - info->taos = pTscObj; + if (pTscObj) { + info->taos = pTscObj; code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); @@ -1528,7 +1534,7 @@ static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLPr info->dataFormat = true; } - if(request){ + if (request) { info->pRequest = request; info->msgBuf.buf = info->pRequest->msgBuf; info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; @@ -1827,10 +1833,11 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { } pVal->length = (int16_t)strlen(value->valuestring); - if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE){ + if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } - if (pVal->type == TSDB_DATA_TYPE_NCHAR && pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + if (pVal->type == TSDB_DATA_TYPE_NCHAR && + pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } @@ -2058,7 +2065,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { SSmlLineInfo elements = {0}; uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); - int ret = smlParseInfluxString(sql, &elements, &info->msgBuf); + int ret = smlParseInfluxString(sql, &elements, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); return ret; @@ -2314,7 +2321,8 @@ static int32_t smlInsertData(SSmlHandle *info) { (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, - (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->msgBuf.buf, info->msgBuf.len); + (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, + info->msgBuf.buf, info->msgBuf.len); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; @@ -2336,7 +2344,12 @@ static int32_t smlInsertData(SSmlHandle *info) { SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - launchAsyncQuery(info->pRequest, info->pQuery, NULL); + SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); + if (pWrapper == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pWrapper->pRequest = info->pRequest; + launchAsyncQuery(info->pRequest, info->pQuery, NULL, pWrapper); return TSDB_CODE_SUCCESS; } @@ -2417,41 +2430,41 @@ static int smlProcess(SSmlHandle *info, char *lines[], int numLines) { } static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) { -// SCatalog *catalog = NULL; -// int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog); -// if (code != TSDB_CODE_SUCCESS) { -// uError("SML get catalog error %d", code); -// return code; -// } -// -// SName name; -// tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db)); -// char dbFname[TSDB_DB_FNAME_LEN] = {0}; -// tNameGetFullDbName(&name, dbFname); -// SDbCfgInfo pInfo = {0}; -// -// SRequestConnInfo conn = {0}; -// conn.pTrans = taos->pAppInfo->pTransporter; -// conn.requestId = request->requestId; -// conn.requestObjRefId = request->self; -// conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp); -// -// code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo); -// if (code != TSDB_CODE_SUCCESS) { -// return code; -// } -// taosArrayDestroy(pInfo.pRetensions); -// -// if (!pInfo.schemaless) { -// return TSDB_CODE_SML_INVALID_DB_CONF; -// } + // SCatalog *catalog = NULL; + // int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog); + // if (code != TSDB_CODE_SUCCESS) { + // uError("SML get catalog error %d", code); + // return code; + // } + // + // SName name; + // tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db)); + // char dbFname[TSDB_DB_FNAME_LEN] = {0}; + // tNameGetFullDbName(&name, dbFname); + // SDbCfgInfo pInfo = {0}; + // + // SRequestConnInfo conn = {0}; + // conn.pTrans = taos->pAppInfo->pTransporter; + // conn.requestId = request->requestId; + // conn.requestObjRefId = request->self; + // conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp); + // + // code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo); + // if (code != TSDB_CODE_SUCCESS) { + // return code; + // } + // taosArrayDestroy(pInfo.pRetensions); + // + // if (!pInfo.schemaless) { + // return TSDB_CODE_SML_INVALID_DB_CONF; + // } return TSDB_CODE_SUCCESS; } static void smlInsertCallback(void *param, void *res, int32_t code) { SRequestObj *pRequest = (SRequestObj *)res; SSmlHandle *info = (SSmlHandle *)param; - int32_t rows = taos_affected_rows(pRequest); + int32_t rows = taos_affected_rows(pRequest); uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); Params *pParam = info->params; @@ -2461,7 +2474,7 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { if (code != TSDB_CODE_SUCCESS) { pParam->request->code = code; pParam->request->body.resInfo.numOfRows += rows; - }else{ + } else { pParam->request->body.resInfo.numOfRows += info->affectedRows; } if (pParam->cnt == pParam->total) { @@ -2497,20 +2510,20 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { * */ -TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { +TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; } - SRequestObj* request = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); - if(!request){ + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT); + if (!request) { uError("SML:taos_schemaless_insert error request is null"); return NULL; } - int batchs = 0; - STscObj* pTscObj = request->pTscObj; + int batchs = 0; + STscObj *pTscObj = request->pTscObj; pTscObj->schemalessType = 1; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; @@ -2526,7 +2539,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr goto end; } - if(isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS){ + if (isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS) { request->code = TSDB_CODE_SML_INVALID_DB_CONF; smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL); goto end; @@ -2551,9 +2564,9 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr goto end; } - if(protocol == TSDB_SML_JSON_PROTOCOL){ + if (protocol == TSDB_SML_JSON_PROTOCOL) { numLines = 1; - }else if(numLines <= 0){ + } else if (numLines <= 0) { request->code = TSDB_CODE_SML_INVALID_DATA; smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL); goto end; @@ -2562,14 +2575,14 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr batchs = ceil(((double)numLines) / LINE_BATCH); params.total = batchs; for (int i = 0; i < batchs; ++i) { - SRequestObj* req = (SRequestObj*)createRequest(pTscObj->id, TSDB_SQL_INSERT); - if(!req){ + SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT); + if (!req) { request->code = TSDB_CODE_OUT_OF_MEMORY; uError("SML:taos_schemaless_insert error request is null"); goto end; } - SSmlHandle* info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision); - if(!info){ + SSmlHandle *info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision); + if (!info) { request->code = TSDB_CODE_OUT_OF_MEMORY; uError("SML:taos_schemaless_insert error SSmlHandle is null"); goto end; @@ -2599,8 +2612,8 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr end: taosThreadSpinDestroy(¶ms.lock); tsem_destroy(¶ms.sem); -// ((STscObj *)taos)->schemalessType = 0; + // ((STscObj *)taos)->schemalessType = 0; pTscObj->schemalessType = 1; uDebug("resultend:%s", request->msgBuf); - return (TAOS_RES*)request; + return (TAOS_RES *)request; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 110c3bc8f47bc4228539782fc49fbc8bf8927c29..ad5772e0fe534c7c5ac3e795abf476084f734ebb 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -411,7 +411,7 @@ _exit: int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) { int32_t code = 0; - int32_t rLen; + int32_t rLen = 0; TSROW_LEN(pRow, rLen); (*ppRow) = (STSRow2 *)taosMemoryMalloc(rLen); @@ -1678,8 +1678,8 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { int32_t size; ASSERT(pColDataSrc->nVal > 0); - ASSERT(pColDataDest->cid = pColDataSrc->cid); - ASSERT(pColDataDest->type = pColDataSrc->type); + ASSERT(pColDataDest->cid == pColDataSrc->cid); + ASSERT(pColDataDest->type == pColDataSrc->type); pColDataDest->smaOn = pColDataSrc->smaOn; pColDataDest->nVal = pColDataSrc->nVal; @@ -1711,4 +1711,4 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { _exit: return code; -} \ No newline at end of file +} diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ce9a9a7b5099825b731542fc71810b2d4bf867fd..88df54a9eace1c310af0c0e9787d8a8774803e97 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -85,7 +85,8 @@ uint16_t tsTelemPort = 80; char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. // If set to empty system will generate table name using MD5 hash. -bool tsSmlDataFormat = false; // true means that the name and order of cols in each line are the same(only for influx protocol) +// true means that the name and order of cols in each line are the same(only for influx protocol) +bool tsSmlDataFormat = false; // query int32_t tsQueryPolicy = 1; @@ -125,6 +126,9 @@ int32_t tsMaxNumOfDistinctResults = 1000 * 10000; // 1 database precision unit for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; +// maximum memory allowed to be allocated for a single csv load (in MB) +int32_t tsMaxMemUsedByInsert = 1024; + // the maximum allowed query buffer size during query processing for each data node. // -1 no limit (default) // 0 no query allowed, queries are disabled @@ -296,6 +300,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); @@ -374,8 +379,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4); if (cfgAddInt32(pCfg, "numOfVnodeStreamThreads", tsNumOfVnodeStreamThreads, 4, 1024, 0) != 0) return -1; -// tsNumOfVnodeFetchThreads = 1; -// if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1, 0) != 0) return -1; + // tsNumOfVnodeFetchThreads = 1; + // if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1, 0) != 0) return -1; tsNumOfVnodeWriteThreads = tsNumOfCores; tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1); @@ -497,15 +502,15 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } -/* - pItem = cfgGetItem(tsCfg, "numOfVnodeFetchThreads"); - if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfVnodeFetchThreads = numOfCores / 4; - tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); - pItem->i32 = tsNumOfVnodeFetchThreads; - pItem->stype = stype; - } -*/ + /* + pItem = cfgGetItem(tsCfg, "numOfVnodeFetchThreads"); + if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { + tsNumOfVnodeFetchThreads = numOfCores / 4; + tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); + pItem->i32 = tsNumOfVnodeFetchThreads; + pItem->stype = stype; + } + */ pItem = cfgGetItem(tsCfg, "numOfVnodeWriteThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { @@ -648,6 +653,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; + tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; @@ -705,7 +712,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; tsNumOfVnodeStreamThreads = cfgGetItem(pCfg, "numOfVnodeStreamThreads")->i32; -// tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; + // tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32; tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32; tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; @@ -877,6 +884,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) { tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; + } else if (strcasecmp("maxMemUsedByInsert", name) == 0) { + tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; } break; } @@ -955,10 +964,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; } else if (strcasecmp("numOfVnodeQueryThreads", name) == 0) { tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; -/* - } else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) { - tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; -*/ + /* + } else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) { + tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; + */ } else if (strcasecmp("numOfVnodeWriteThreads", name) == 0) { tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32; } else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) { diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index c86bf08e8168f00f882333878e11af90a0e65627..3dd865e91ff55851b5ed8353aa111e5b3ec08d74 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -35,7 +35,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { // tscError("tmp file created failed since %s", terrstr()); return NULL; } - + STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf)); if (pTSBuf == NULL) { return NULL; @@ -52,10 +52,13 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } if (!autoDelete) { - taosRemoveFile(pTSBuf->path); + if (taosRemoveFile(pTSBuf->path) != 0) { + taosMemoryFree(pTSBuf); + return NULL; + } } - if (NULL == allocResForTSBuf(pTSBuf)) { + if (allocResForTSBuf(pTSBuf) == NULL) { return NULL; } diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 45bef7f98ea3f590f53c646cc2abdebb6c5091dc..581686ba904001b80638efd9ab225c3f68e91e09 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -14,7 +14,4 @@ target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" ) -IF (TD_LINUX_64 AND JEMALLOC_ENABLED) - add_dependencies(taosd jemalloc) -ENDIF () target_link_libraries(taosd dnode) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index b882e02e946353a1916bf3bdbe3e154ba1de04b5..a49e855e39ec1f635e48dd9c47b9a4f942864655 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -201,14 +201,13 @@ _OVER: taosCloseFile(&pFile); taosMemoryFree(content); - for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = ppVnodes[i]; - if (pVnode != NULL) { - vmReleaseVnode(pMgmt, pVnode); - } - } - if (ppVnodes != NULL) { + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = ppVnodes[i]; + if (pVnode != NULL) { + vmReleaseVnode(pMgmt, pVnode); + } + } taosMemoryFree(ppVnodes); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 4047bc2340de74500f9c675b6c7ee570f6b94b6a..d8456ad6264c5c784be91afdcd8c58b832a99ee8 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" -void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { +void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) { pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad)); if (pInfo->pVloads == NULL) return; @@ -30,6 +30,7 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { SVnodeObj *pVnode = *ppVnode; SVnodeLoad vload = {0}; vnodeGetLoad(pVnode->pImpl, &vload); + if (isReset) vnodeResetLoad(pVnode->pImpl, &vload); taosArrayPush(pInfo->pVloads, &vload); pIter = taosHashIterate(pMgmt->hash, pIter); } @@ -39,7 +40,7 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) { SMonVloadInfo vloads = {0}; - vmGetVnodeLoads(pMgmt, &vloads); + vmGetVnodeLoads(pMgmt, &vloads, true); SArray *pVloads = vloads.pVloads; if (pVloads == NULL) return; @@ -66,10 +67,10 @@ void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) { pInfo->vstat.totalVnodes = totalVnodes; pInfo->vstat.masterNum = masterNum; pInfo->vstat.numOfSelectReqs = numOfSelectReqs - pMgmt->state.numOfSelectReqs; - pInfo->vstat.numOfInsertReqs = numOfInsertReqs - pMgmt->state.numOfInsertReqs; - pInfo->vstat.numOfInsertSuccessReqs = numOfInsertSuccessReqs - pMgmt->state.numOfInsertSuccessReqs; - pInfo->vstat.numOfBatchInsertReqs = numOfBatchInsertReqs - pMgmt->state.numOfBatchInsertReqs; - pInfo->vstat.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs - pMgmt->state.numOfBatchInsertSuccessReqs; + pInfo->vstat.numOfInsertReqs = numOfInsertReqs; // delta + pInfo->vstat.numOfInsertSuccessReqs = numOfInsertSuccessReqs; // delta + pInfo->vstat.numOfBatchInsertReqs = numOfBatchInsertReqs; // delta + pInfo->vstat.numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; // delta pMgmt->state.totalVnodes = totalVnodes; pMgmt->state.masterNum = masterNum; pMgmt->state.numOfSelectReqs = numOfSelectReqs; @@ -109,7 +110,7 @@ int32_t vmProcessGetMonitorInfoReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmProcessGetLoadsReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SMonVloadInfo vloads = {0}; - vmGetVnodeLoads(pMgmt, &vloads); + vmGetVnodeLoads(pMgmt, &vloads, false); int32_t rspLen = tSerializeSMonVloadInfo(NULL, 0, &vloads); if (rspLen < 0) { @@ -212,38 +213,47 @@ static int32_t vmTsmaProcessCreate(SVnode *pVnode, SCreateVnodeReq *pReq) { } int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - SCreateVnodeReq createReq = {0}; + SCreateVnodeReq req = {0}; SVnodeCfg vnodeCfg = {0}; SWrapperCfg wrapperCfg = {0}; int32_t code = -1; char path[TSDB_FILENAME_LEN] = {0}; - if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { + if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &req) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } dInfo( - "vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d " - "tsdbPageSize:%d", - createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize, - createReq.sstTrigger, createReq.tsdbPageSize); - dInfo("vgId:%d, hashMethod:%d begin:%u end:%u prefix:%d surfix:%d", createReq.vgId, createReq.hashMethod, - createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix); - vmGenerateVnodeCfg(&createReq, &vnodeCfg); - - if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) { - dError("vgId:%d, failed to adjust tsma days since %s", createReq.vgId, terrstr()); + "vgId:%d, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 + " cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64 + "days:%d keep0:%d keep1:%d keep2:%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d, wal " + "fsync:%d level:%d retentionPeriod:%d retentionSize:%d rollPeriod:%d segSize:%d, hash method:%d begin:%u end:%u " + "prefix:%d surfix:%d replica:%d selfIndex:%d strict:%d", + req.vgId, req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024, + req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, req.tsdbPageSize * 1024, req.db, req.dbUid, + req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.isTsma, req.precision, req.compression, + req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize, + req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, + req.replica, req.selfIndex, req.strict); + for (int32_t i = 0; i < req.replica; ++i) { + dInfo("vgId:%d, replica:%d fqdn:%s port:%u", req.vgId, req.replicas[i].id, req.replicas[i].fqdn, + req.replicas[i].port); + } + vmGenerateVnodeCfg(&req, &vnodeCfg); + + if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) { + dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, terrstr()); code = terrno; goto _OVER; } - vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg); + vmGenerateWrapperCfg(pMgmt, &req, &wrapperCfg); - SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); if (pVnode != NULL) { - dDebug("vgId:%d, already exist", createReq.vgId); - tFreeSCreateVnodeReq(&createReq); + dDebug("vgId:%d, already exist", req.vgId); + tFreeSCreateVnodeReq(&req); vmReleaseVnode(pMgmt, pVnode); terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; code = terrno; @@ -252,36 +262,36 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) { - tFreeSCreateVnodeReq(&createReq); - dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); + tFreeSCreateVnodeReq(&req); + dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr()); code = terrno; goto _OVER; } SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { - dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); + dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr()); code = terrno; goto _OVER; } code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl); if (code != 0) { - dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); + dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr()); code = terrno; goto _OVER; } - code = vmTsmaProcessCreate(pImpl, &createReq); + code = vmTsmaProcessCreate(pImpl, &req); if (code != 0) { - dError("vgId:%d, failed to create tsma since %s", createReq.vgId, terrstr()); + dError("vgId:%d, failed to create tsma since %s", req.vgId, terrstr()); code = terrno; goto _OVER; } code = vnodeStart(pImpl); if (code != 0) { - dError("vgId:%d, failed to start sync since %s", createReq.vgId, terrstr()); + dError("vgId:%d, failed to start sync since %s", req.vgId, terrstr()); goto _OVER; } @@ -296,10 +306,10 @@ _OVER: vnodeClose(pImpl); vnodeDestroy(path, pMgmt->pTfs); } else { - dInfo("vgId:%d, vnode is created", createReq.vgId); + dInfo("vgId:%d, vnode is created", req.vgId); } - tFreeSCreateVnodeReq(&createReq); + tFreeSCreateVnodeReq(&req); terrno = code; return code; } diff --git a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h index 8c2d57808fc5d8e29c4bef5079f504c8a9e39802..d3f1044f8804dbb83a8aea3f2864246f976dbde5 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h @@ -35,7 +35,7 @@ void qmGetMonitorInfo(void *pMgmt, SMonQmInfo *pInfo); void smGetMonitorInfo(void *pMgmt, SMonSmInfo *pInfo); void bmGetMonitorInfo(void *pMgmt, SMonBmInfo *pInfo); -void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo); +void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo, bool isReset); void mmGetMnodeLoads(void *pMgmt, SMonMloadInfo *pInfo); void qmGetQnodeLoads(void *pMgmt, SQnodeLoad *pInfo); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index ecad390ef94a635fdeed8256004fce9978fde822..50d6aca53e56093db5201e7eabee6df17bdd3e36 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -152,7 +152,7 @@ void dmGetVnodeLoads(SMonVloadInfo *pInfo) { if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_VM_LOAD, tDeserializeSMonVloadInfo, pInfo); } else if (pWrapper->pMgmt != NULL) { - vmGetVnodeLoads(pWrapper->pMgmt, pInfo); + vmGetVnodeLoads(pWrapper->pMgmt, pInfo, false); } dmReleaseWrapper(pWrapper); } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index c8237d17d8e37808f82c4569ee235e9d1ac5d45a..ad5fbef34cc9b94273d10cd4be88c17d1c940182 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -44,7 +44,8 @@ int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pV int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo); int32_t mndSetMoveVgroupInfoToTrans(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vn, SArray *pArray); int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId); -int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray); +int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup, + SArray *pArray); void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *cntlen, bool standby); void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index adc5ff2b42e9190e02c07a05db2d30eecd8861e1..6670211d7834456af6ed4456881dea149e4e0414 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -746,7 +746,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { +static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; SArray *pArray = mndBuildDnodesArray(pMnode, 0); @@ -756,8 +756,8 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; - if (mndVgroupInDb(pVgroup, pNew->uid)) { - if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup, pArray) != 0) { + if (mndVgroupInDb(pVgroup, pNewDb->uid)) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pOldDb, pNewDb, pVgroup, pArray) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); taosArrayDestroy(pArray); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 059bfdeeb322158f6af66340cfaf4fedb24601a1..462b068a738b5ca95fc77b8ca86186b9d37c3e1a 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -117,9 +117,13 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { SArray *pArray = taosArrayInit(innerSz, sizeof(void *)); for (int32_t j = 0; j < innerSz; j++) { SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); - if (pTask == NULL) return -1; + if (pTask == NULL) { + taosArrayDestroy(pArray); + return -1; + } if (tDecodeSStreamTask(pDecoder, pTask) < 0) { taosMemoryFree(pTask); + taosArrayDestroy(pArray); return -1; } taosArrayPush(pArray, &pTask); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 4b76909a96670833ccd2c1f35778f0fc5155848b..826e1d2fd0d8d1bd5936f8a22a1c2e195e29dcf2 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -87,7 +87,7 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj); if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); + (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); mInfo("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a31bf6b8e24201d5923a958bf4fb21c88d492eea..8e91ec90f537cd069f87474e29695eaa37ce30c0 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -405,7 +405,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 pDnode->memUsed = mndGetVnodesMemory(pMnode, pDnode->id); mInfo("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d memory avail:%" PRId64 " used:%" PRId64, pDnode->id, - pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online, pDnode->memAvail, pDnode->memUsed); + pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online, pDnode->memAvail, pDnode->memUsed); if (isMnode) { pDnode->numOfVnodes++; @@ -1293,7 +1293,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, mError("db:%s, vgId:%d, no enough memory:%" PRId64 " in dnode:%d avail:%" PRId64 " used:%" PRId64, pVgroup->dbName, pVgroup->vgId, vgMem, pNew3->id, pNew3->memAvail, pNew3->memUsed); terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; - goto _OVER; + goto _OVER; } else { pNew3->memUsed += vgMem; } @@ -1530,67 +1530,102 @@ _OVER: #endif } -int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { - if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) { - return -1; +static int32_t mndCheckDnodeMemory(SMnode *pMnode, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pOldVgroup, + SVgObj *pNewVgroup, SArray *pArray) { + for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + bool inVgroup = false; + for (int32_t j = 0; j < pOldVgroup->replica; ++j) { + SVnodeGid *pVgId = &pOldVgroup->vnodeGid[i]; + if (pDnode->id == pVgId->dnodeId) { + pDnode->memUsed -= mndGetVgroupMemory(pMnode, pOldDb, pOldVgroup); + inVgroup = true; + } } - } else { - SVgObj newVgroup = {0}; - memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); - mndTransSetSerial(pTrans); - - if (newVgroup.replica < pDb->cfg.replications) { - mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId, - pVgroup->vnodeGid[0].dnodeId); - - if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - - if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - } else if (newVgroup.replica > pDb->cfg.replications) { - mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); - - SVnodeGid del1 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1; - if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del1, true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - - SVnodeGid del2 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1; - if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del2, true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + for (int32_t j = 0; j < pNewVgroup->replica; ++j) { + SVnodeGid *pVgId = &pNewVgroup->vnodeGid[i]; + if (pDnode->id == pVgId->dnodeId) { + pDnode->memUsed += mndGetVgroupMemory(pMnode, pNewDb, pNewVgroup); + inVgroup = true; + } + } + if (pDnode->memAvail - pDnode->memUsed <= 0) { + mError("db:%s, vgId:%d, no enough memory in dnode:%d, avail:%" PRId64 " used:%" PRId64, pNewVgroup->dbName, + pNewVgroup->vgId, pDnode->id, pDnode->memAvail, pDnode->memUsed); + terrno = TSDB_CODE_MND_NO_ENOUGH_MEM_IN_DNODE; + return -1; + } else if (inVgroup) { + mInfo("db:%s, vgId:%d, memory in dnode:%d, avail:%" PRId64 " used:%" PRId64, pNewVgroup->dbName, + pNewVgroup->vgId, pDnode->id, pDnode->memAvail, pDnode->memUsed); } else { } + } + return 0; +} - { - SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { - sdbFreeRaw(pVgRaw); - return -1; - } - (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); +int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup, + SArray *pArray) { + SVgObj newVgroup = {0}; + memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); + + if (pVgroup->replica <= 0 || pVgroup->replica == pNewDb->cfg.replications) { + if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) return -1; + if (mndCheckDnodeMemory(pMnode, pOldDb, pNewDb, &newVgroup, pVgroup, pArray) != 0) return -1; + return 0; + } + + mndTransSetSerial(pTrans); + + if (newVgroup.replica < pNewDb->cfg.replications) { + mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId, + pVgroup->vnodeGid[0].dnodeId); + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + } else if (newVgroup.replica > pNewDb->cfg.replications) { + mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); + + SVnodeGid del1 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1; + if (mndAddSetVnodeStandByAction(pMnode, pTrans, pNewDb, pVgroup, &del1, true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del1, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + + SVnodeGid del2 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1; + if (mndAddSetVnodeStandByAction(pMnode, pTrans, pNewDb, pVgroup, &del2, true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del2, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + } else { + } + + { + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; } + (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + } - { - SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { - sdbFreeRaw(pVgRaw); - return -1; - } - (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + { + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; } + (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); } return 0; @@ -1650,8 +1685,8 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg2, TDMT_VND_ALTER_HASHRANGE) != 0) goto _OVER; // adjust vgroup - if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg1, pArray) != 0) goto _OVER; - if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg2, pArray) != 0) goto _OVER; + if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER; + if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER; _OVER: mndTransDrop(pTrans); @@ -1784,7 +1819,7 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); mInfo("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes, - pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes); + pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes); } SDnodeObj *pSrc = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1); @@ -1793,7 +1828,7 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes; float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes; mInfo("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore, - pDst->id, dstScore); + pDst->id, dstScore); if (srcScore > dstScore - 0.000001) { code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst, pBalancedVgroups); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 9926c5a8c414ffdc684805d7b3efaa6dc9759fd7..d006f12a3f393b7ed562de56a9b14072fa81f260 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -73,6 +73,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num); int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num); int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num); +void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName); diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index adfbb919206a184664655cc11746c1d25c14147b..dbe2f80150a021c29e2b7c02153675ec63d332e6 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -67,6 +67,10 @@ void metaCacheClose(SMeta* pMeta); int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo); int32_t metaCacheDrop(SMeta* pMeta, int64_t uid); +int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo); +int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid); +int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); + struct SMeta { TdThreadRwlock lock; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 2a8a74d297d6f65e45e5b40a2b25555513c31520..712f8bd15b9b3e189ba7649a486743d284c9fd15 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -56,6 +56,7 @@ typedef struct SSma SSma; typedef struct STsdb STsdb; typedef struct STQ STQ; typedef struct SVState SVState; +typedef struct SVStatis SVStatis; typedef struct SVBufPool SVBufPool; typedef struct SQWorker SQHandle; typedef struct STsdbKeepCfg STsdbKeepCfg; @@ -141,6 +142,12 @@ typedef struct SMetaInfo { } SMetaInfo; int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo); +typedef struct { + int64_t uid; + int64_t ctbNum; +} SMetaStbStats; +int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); + // tsdb int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg); int tsdbClose(STsdb** pTsdb); @@ -284,9 +291,17 @@ struct SVState { int64_t commitTerm; }; +struct SVStatis { + int64_t nInsert; // delta + int64_t nInsertSuccess; // delta + int64_t nBatchInsert; // delta + int64_t nBatchInsertSuccess; // delta +}; + struct SVnodeInfo { SVnodeCfg config; SVState state; + SVStatis statis; }; typedef enum { @@ -309,6 +324,7 @@ struct SVnode { char* path; SVnodeCfg config; SVState state; + SVStatis statis; STfs* pTfs; SMsgCb msgCb; TdThreadMutex mutex; diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index b8cc9f0df2f10d6bd42f6d68f801d2a749479626..6f19d43e6943996c52a24a4b1b9e90b2ceeb7a52 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -14,7 +14,8 @@ */ #include "meta.h" -#define META_CACHE_BASE_BUCKET 1024 +#define META_CACHE_BASE_BUCKET 1024 +#define META_CACHE_STATS_BUCKET 16 // (uid , suid) : child table // (uid, 0) : normal table @@ -25,12 +26,59 @@ struct SMetaCacheEntry { SMetaInfo info; }; +typedef struct SMetaStbStatsEntry { + struct SMetaStbStatsEntry* next; + SMetaStbStats info; +} SMetaStbStatsEntry; + struct SMetaCache { - int32_t nEntry; - int32_t nBucket; - SMetaCacheEntry** aBucket; + // child, normal, super, table entry cache + struct SEntryCache { + int32_t nEntry; + int32_t nBucket; + SMetaCacheEntry** aBucket; + } sEntryCache; + + // stable stats cache + struct SStbStatsCache { + int32_t nEntry; + int32_t nBucket; + SMetaStbStatsEntry** aBucket; + } sStbStatsCache; + + // query cache }; +static void entryCacheClose(SMeta* pMeta) { + if (pMeta->pCache) { + // close entry cache + for (int32_t iBucket = 0; iBucket < pMeta->pCache->sEntryCache.nBucket; iBucket++) { + SMetaCacheEntry* pEntry = pMeta->pCache->sEntryCache.aBucket[iBucket]; + while (pEntry) { + SMetaCacheEntry* tEntry = pEntry->next; + taosMemoryFree(pEntry); + pEntry = tEntry; + } + } + taosMemoryFree(pMeta->pCache->sEntryCache.aBucket); + } +} + +static void statsCacheClose(SMeta* pMeta) { + if (pMeta->pCache) { + // close entry cache + for (int32_t iBucket = 0; iBucket < pMeta->pCache->sStbStatsCache.nBucket; iBucket++) { + SMetaStbStatsEntry* pEntry = pMeta->pCache->sStbStatsCache.aBucket[iBucket]; + while (pEntry) { + SMetaStbStatsEntry* tEntry = pEntry->next; + taosMemoryFree(pEntry); + pEntry = tEntry; + } + } + taosMemoryFree(pMeta->pCache->sStbStatsCache.aBucket); + } +} + int32_t metaCacheOpen(SMeta* pMeta) { int32_t code = 0; SMetaCache* pCache = NULL; @@ -41,36 +89,45 @@ int32_t metaCacheOpen(SMeta* pMeta) { goto _err; } - pCache->nEntry = 0; - pCache->nBucket = META_CACHE_BASE_BUCKET; - pCache->aBucket = (SMetaCacheEntry**)taosMemoryCalloc(pCache->nBucket, sizeof(SMetaCacheEntry*)); - if (pCache->aBucket == NULL) { + // open entry cache + pCache->sEntryCache.nEntry = 0; + pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET; + pCache->sEntryCache.aBucket = + (SMetaCacheEntry**)taosMemoryCalloc(pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*)); + if (pCache->sEntryCache.aBucket == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pCache); goto _err; } + // open stats cache + pCache->sStbStatsCache.nEntry = 0; + pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET; + pCache->sStbStatsCache.aBucket = + (SMetaStbStatsEntry**)taosMemoryCalloc(pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*)); + if (pCache->sStbStatsCache.aBucket == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err2; + } + pMeta->pCache = pCache; _exit: return code; +_err2: + entryCacheClose(pMeta); + _err: + taosMemoryFree(pCache); + metaError("vgId:%d meta open cache failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); return code; } void metaCacheClose(SMeta* pMeta) { if (pMeta->pCache) { - for (int32_t iBucket = 0; iBucket < pMeta->pCache->nBucket; iBucket++) { - SMetaCacheEntry* pEntry = pMeta->pCache->aBucket[iBucket]; - while (pEntry) { - SMetaCacheEntry* tEntry = pEntry->next; - taosMemoryFree(pEntry); - pEntry = tEntry; - } - } - taosMemoryFree(pMeta->pCache->aBucket); + entryCacheClose(pMeta); + statsCacheClose(pMeta); taosMemoryFree(pMeta->pCache); pMeta->pCache = NULL; } @@ -81,9 +138,9 @@ static int32_t metaRehashCache(SMetaCache* pCache, int8_t expand) { int32_t nBucket; if (expand) { - nBucket = pCache->nBucket * 2; + nBucket = pCache->sEntryCache.nBucket * 2; } else { - nBucket = pCache->nBucket / 2; + nBucket = pCache->sEntryCache.nBucket / 2; } SMetaCacheEntry** aBucket = (SMetaCacheEntry**)taosMemoryCalloc(nBucket, sizeof(SMetaCacheEntry*)); @@ -93,8 +150,8 @@ static int32_t metaRehashCache(SMetaCache* pCache, int8_t expand) { } // rehash - for (int32_t iBucket = 0; iBucket < pCache->nBucket; iBucket++) { - SMetaCacheEntry* pEntry = pCache->aBucket[iBucket]; + for (int32_t iBucket = 0; iBucket < pCache->sEntryCache.nBucket; iBucket++) { + SMetaCacheEntry* pEntry = pCache->sEntryCache.aBucket[iBucket]; while (pEntry) { SMetaCacheEntry* pTEntry = pEntry->next; @@ -107,9 +164,9 @@ static int32_t metaRehashCache(SMetaCache* pCache, int8_t expand) { } // final set - taosMemoryFree(pCache->aBucket); - pCache->nBucket = nBucket; - pCache->aBucket = aBucket; + taosMemoryFree(pCache->sEntryCache.aBucket); + pCache->sEntryCache.nBucket = nBucket; + pCache->sEntryCache.aBucket = aBucket; _exit: return code; @@ -122,8 +179,8 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { // search SMetaCache* pCache = pMeta->pCache; - int32_t iBucket = TABS(pInfo->uid) % pCache->nBucket; - SMetaCacheEntry** ppEntry = &pCache->aBucket[iBucket]; + int32_t iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket; + SMetaCacheEntry** ppEntry = &pCache->sEntryCache.aBucket[iBucket]; while (*ppEntry && (*ppEntry)->info.uid != pInfo->uid) { ppEntry = &(*ppEntry)->next; } @@ -135,11 +192,11 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { (*ppEntry)->info.skmVer = pInfo->skmVer; } } else { // insert - if (pCache->nEntry >= pCache->nBucket) { + if (pCache->sEntryCache.nEntry >= pCache->sEntryCache.nBucket) { code = metaRehashCache(pCache, 1); if (code) goto _exit; - iBucket = TABS(pInfo->uid) % pCache->nBucket; + iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket; } SMetaCacheEntry* pEntryNew = (SMetaCacheEntry*)taosMemoryMalloc(sizeof(*pEntryNew)); @@ -149,9 +206,9 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { } pEntryNew->info = *pInfo; - pEntryNew->next = pCache->aBucket[iBucket]; - pCache->aBucket[iBucket] = pEntryNew; - pCache->nEntry++; + pEntryNew->next = pCache->sEntryCache.aBucket[iBucket]; + pCache->sEntryCache.aBucket[iBucket] = pEntryNew; + pCache->sEntryCache.nEntry++; } _exit: @@ -162,8 +219,8 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid) { int32_t code = 0; SMetaCache* pCache = pMeta->pCache; - int32_t iBucket = TABS(uid) % pCache->nBucket; - SMetaCacheEntry** ppEntry = &pCache->aBucket[iBucket]; + int32_t iBucket = TABS(uid) % pCache->sEntryCache.nBucket; + SMetaCacheEntry** ppEntry = &pCache->sEntryCache.aBucket[iBucket]; while (*ppEntry && (*ppEntry)->info.uid != uid) { ppEntry = &(*ppEntry)->next; } @@ -172,8 +229,9 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid) { if (pEntry) { *ppEntry = pEntry->next; taosMemoryFree(pEntry); - pCache->nEntry--; - if (pCache->nEntry < pCache->nBucket / 4 && pCache->nBucket > META_CACHE_BASE_BUCKET) { + pCache->sEntryCache.nEntry--; + if (pCache->sEntryCache.nEntry < pCache->sEntryCache.nBucket / 4 && + pCache->sEntryCache.nBucket > META_CACHE_BASE_BUCKET) { code = metaRehashCache(pCache, 0); if (code) goto _exit; } @@ -189,8 +247,134 @@ int32_t metaCacheGet(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo) { int32_t code = 0; SMetaCache* pCache = pMeta->pCache; - int32_t iBucket = TABS(uid) % pCache->nBucket; - SMetaCacheEntry* pEntry = pCache->aBucket[iBucket]; + int32_t iBucket = TABS(uid) % pCache->sEntryCache.nBucket; + SMetaCacheEntry* pEntry = pCache->sEntryCache.aBucket[iBucket]; + + while (pEntry && pEntry->info.uid != uid) { + pEntry = pEntry->next; + } + + if (pEntry) { + *pInfo = pEntry->info; + } else { + code = TSDB_CODE_NOT_FOUND; + } + + return code; +} + +static int32_t metaRehashStatsCache(SMetaCache* pCache, int8_t expand) { + int32_t code = 0; + int32_t nBucket; + + if (expand) { + nBucket = pCache->sStbStatsCache.nBucket * 2; + } else { + nBucket = pCache->sStbStatsCache.nBucket / 2; + } + + SMetaStbStatsEntry** aBucket = (SMetaStbStatsEntry**)taosMemoryCalloc(nBucket, sizeof(SMetaStbStatsEntry*)); + if (aBucket == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + // rehash + for (int32_t iBucket = 0; iBucket < pCache->sStbStatsCache.nBucket; iBucket++) { + SMetaStbStatsEntry* pEntry = pCache->sStbStatsCache.aBucket[iBucket]; + + while (pEntry) { + SMetaStbStatsEntry* pTEntry = pEntry->next; + + pEntry->next = aBucket[TABS(pEntry->info.uid) % nBucket]; + aBucket[TABS(pEntry->info.uid) % nBucket] = pEntry; + + pEntry = pTEntry; + } + } + + // final set + taosMemoryFree(pCache->sStbStatsCache.aBucket); + pCache->sStbStatsCache.nBucket = nBucket; + pCache->sStbStatsCache.aBucket = aBucket; + +_exit: + return code; +} + +int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) { + int32_t code = 0; + + // ASSERT(metaIsWLocked(pMeta)); + + // search + SMetaCache* pCache = pMeta->pCache; + int32_t iBucket = TABS(pInfo->uid) % pCache->sStbStatsCache.nBucket; + SMetaStbStatsEntry** ppEntry = &pCache->sStbStatsCache.aBucket[iBucket]; + while (*ppEntry && (*ppEntry)->info.uid != pInfo->uid) { + ppEntry = &(*ppEntry)->next; + } + + if (*ppEntry) { // update + (*ppEntry)->info.ctbNum = pInfo->ctbNum; + } else { // insert + if (pCache->sStbStatsCache.nEntry >= pCache->sStbStatsCache.nBucket) { + code = metaRehashStatsCache(pCache, 1); + if (code) goto _exit; + + iBucket = TABS(pInfo->uid) % pCache->sStbStatsCache.nBucket; + } + + SMetaStbStatsEntry* pEntryNew = (SMetaStbStatsEntry*)taosMemoryMalloc(sizeof(*pEntryNew)); + if (pEntryNew == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pEntryNew->info = *pInfo; + pEntryNew->next = pCache->sStbStatsCache.aBucket[iBucket]; + pCache->sStbStatsCache.aBucket[iBucket] = pEntryNew; + pCache->sStbStatsCache.nEntry++; + } + +_exit: + return code; +} + +int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid) { + int32_t code = 0; + + SMetaCache* pCache = pMeta->pCache; + int32_t iBucket = TABS(uid) % pCache->sStbStatsCache.nBucket; + SMetaStbStatsEntry** ppEntry = &pCache->sStbStatsCache.aBucket[iBucket]; + while (*ppEntry && (*ppEntry)->info.uid != uid) { + ppEntry = &(*ppEntry)->next; + } + + SMetaStbStatsEntry* pEntry = *ppEntry; + if (pEntry) { + *ppEntry = pEntry->next; + taosMemoryFree(pEntry); + pCache->sStbStatsCache.nEntry--; + if (pCache->sStbStatsCache.nEntry < pCache->sStbStatsCache.nBucket / 4 && + pCache->sStbStatsCache.nBucket > META_CACHE_STATS_BUCKET) { + code = metaRehashStatsCache(pCache, 0); + if (code) goto _exit; + } + } else { + code = TSDB_CODE_NOT_FOUND; + } + +_exit: + return code; +} + +int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + + SMetaCache* pCache = pMeta->pCache; + int32_t iBucket = TABS(uid) % pCache->sStbStatsCache.nBucket; + SMetaStbStatsEntry* pEntry = pCache->sStbStatsCache.aBucket[iBucket]; while (pEntry && pEntry->info.uid != uid) { pEntry = pEntry->next; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 1b5d25974e14e0facf3eff9e64c68556afa554ff..b4fc2c010b251d4a239179749f2c87cffe4ae635 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1260,3 +1260,32 @@ _exit: tdbFree(pData); return code; } + +int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo) { + int32_t code = 0; + + metaRLock(pMeta); + + // fast path: search cache + if (metaStatsCacheGet(pMeta, uid, pInfo) == TSDB_CODE_SUCCESS) { + metaULock(pMeta); + goto _exit; + } + + // slow path: search TDB + int64_t ctbNum = 0; + vnodeGetCtbNum(pMeta->pVnode, uid, &ctbNum); + + metaULock(pMeta); + + pInfo->uid = uid; + pInfo->ctbNum = ctbNum; + + // upsert the cache + metaWLock(pMeta); + metaStatsCacheUpsert(pMeta, pInfo); + metaULock(pMeta); + +_exit: + return code; +} diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 361e453303f2f5f0cb93164ee301def52dd3ba6d..c8a4ff945d0cc1771b0f074b91ee55887a24fe51 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -362,6 +362,8 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // update uid index metaUpdateUidIdx(pMeta, &nStbEntry); + metaStatsCacheDrop(pMeta, nStbEntry.uid); + metaULock(pMeta); if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); @@ -615,6 +617,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), &pMeta->txn); // drop schema.db (todo) + metaStatsCacheDrop(pMeta, uid); --pMeta->pVnode->config.vndStats.numOfSTables; } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index c29541873e4fcc99323e3d61ab9f100187fe4d25..1a57a391b13d6c51abb95fb48f8e0e7bef0d52b8 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -180,6 +180,7 @@ int32_t tqMetaRestoreCheckInfo(STQ* pTq) { return -1; } } + tdbFree(pKey); tdbTbcClose(pCur); return 0; } diff --git a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c index 084959af6556263ed6b34045cef406bfbc661a47..b63ff8af1d623aa2017ce5768a80e2dbf783d5a4 100644 --- a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c +++ b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c @@ -54,7 +54,7 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) { char* fname = tqOffsetBuildFName(pReader->pTq->path, 0); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); - if (pFile != NULL) { + if (pFile == NULL) { taosMemoryFree(fname); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 0d57c7bb74d6d5822d418338dee558933f7783d0..d10a07969a2b8a325c4a6d7bc9d142515194b881 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -15,6 +15,12 @@ #include "vnd.h" +#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType) \ + do { \ + int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal)); \ + ASSERT(newVal >= 0); \ + } while (0) + int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, TD_VID(pVnode), (void **)&pVnode->pQuery, &pVnode->msgCb); } @@ -375,13 +381,28 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->compStorage = (int64_t)2 * 1073741824; pLoad->pointsWritten = 100; pLoad->numOfSelectReqs = 1; - pLoad->numOfInsertReqs = 3; - pLoad->numOfInsertSuccessReqs = 2; - pLoad->numOfBatchInsertReqs = 5; - pLoad->numOfBatchInsertSuccessReqs = 4; + pLoad->numOfInsertReqs = atomic_load_64(&pVnode->statis.nInsert); + pLoad->numOfInsertSuccessReqs = atomic_load_64(&pVnode->statis.nInsertSuccess); + pLoad->numOfBatchInsertReqs = atomic_load_64(&pVnode->statis.nBatchInsert); + pLoad->numOfBatchInsertSuccessReqs = atomic_load_64(&pVnode->statis.nBatchInsertSuccess); return 0; } +/** + * @brief Reset the statistics value by monitor interval + * + * @param pVnode + * @param pLoad + */ +void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { + VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsert, pLoad->numOfInsertReqs, 64); + VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsertSuccess, pLoad->numOfInsertSuccessReqs, 64); + VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsert, pLoad->numOfBatchInsertReqs, 64); + VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64); + + +} + void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) { if (dbname) { *dbname = pVnode->config.dbname; @@ -483,27 +504,36 @@ static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { } int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { - SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, 0); - if (!pCur) { + SArray *suidList = NULL; + + if (!(suidList = taosArrayInit(1, sizeof(tb_uid_t)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + if (vnodeGetStbIdList(pVnode, 0, suidList) < 0) { + qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr()); + taosArrayDestroy(suidList); return TSDB_CODE_FAILED; } *num = 0; - while (1) { - tb_uid_t id = metaStbCursorNext(pCur); - if (id == 0) { - break; - } + int64_t arrSize = taosArrayGetSize(suidList); + for (int64_t i = 0; i < arrSize; ++i) { + tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + + SMetaStbStats stats = {0}; + metaGetStbStats(pVnode->pMeta, suid, &stats); + int64_t ctbNum = stats.ctbNum; + // vnodeGetCtbNum(pVnode, id, &ctbNum); - int64_t ctbNum = 0; - vnodeGetCtbNum(pVnode, id, &ctbNum); int numOfCols = 0; - vnodeGetStbColumnNum(pVnode, id, &numOfCols); + vnodeGetStbColumnNum(pVnode, suid, &numOfCols); *num += ctbNum * (numOfCols - 1); } - metaCloseStbCursor(pCur); + taosArrayDestroy(suidList); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 5c8590c7c9b88fa18b9002cf6def36d61c6c3f7a..2482ca1a5a5a0bd8d1075508f09d32f4eb580362 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -812,10 +812,10 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock; + SSubmitBlk *pBlock = NULL; SVCreateTbReq createTbReq = {0}; SDecoder decoder = {0}; - int32_t nRows; + int32_t nRows = 0; int32_t tsize, ret; SEncoder encoder = {0}; SArray *newTbUids = NULL; @@ -823,6 +823,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq pRsp->code = 0; pSubmitReq->version = version; + atomic_fetch_add_64(&pVnode->statis.nBatchInsert, 1); #ifdef TD_DEBUG_PRINT_ROW vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); @@ -942,11 +943,15 @@ _exit: taosArrayDestroyEx(submitRsp.pArray, tFreeSSubmitBlkRsp); + atomic_fetch_add_64(&pVnode->statis.nInsert, submitRsp.numOfRows); + atomic_fetch_add_64(&pVnode->statis.nInsertSuccess, submitRsp.affectedRows); + // TODO: the partial success scenario and the error case // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level // 1/level 2. // TODO: refactor if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) { + atomic_fetch_add_64(&pVnode->statis.nBatchInsertSuccess, 1); tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT); } @@ -1024,71 +1029,75 @@ static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, vo } static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SAlterVnodeReq alterReq = {0}; + SAlterVnodeReq req = {0}; bool walChanged = false; bool tsdbChanged = false; - if (tDeserializeSAlterVnodeReq(pReq, len, &alterReq) != 0) { + if (tDeserializeSAlterVnodeReq(pReq, len, &req) != 0) { terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG; } - vInfo("vgId:%d, start to alter vnode config, cacheLast:%d cacheLastSize:%d", TD_VID(pVnode), alterReq.cacheLast, - alterReq.cacheLastSize); - if (pVnode->config.cacheLastSize != alterReq.cacheLastSize) { - pVnode->config.cacheLastSize = alterReq.cacheLastSize; + vInfo("vgId:%d, start to alter vnode config, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 + " cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d fsync:%d level:%d strict:%d", + TD_VID(pVnode), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024, + req.cacheLast, req.cacheLastSize, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, + req.walFsyncPeriod, req.walLevel, req.strict); + + if (pVnode->config.cacheLastSize != req.cacheLastSize) { + pVnode->config.cacheLastSize = req.cacheLastSize; tsdbCacheSetCapacity(pVnode, (size_t)pVnode->config.cacheLastSize * 1024 * 1024); } - if (pVnode->config.szBuf != alterReq.buffer * 1024LL * 1024LL) { + if (pVnode->config.szBuf != req.buffer * 1024LL * 1024LL) { vInfo("vgId:%d vnode buffer is changed from %" PRId64 " to %" PRId64, TD_VID(pVnode), pVnode->config.szBuf, - alterReq.buffer * 1024LL * 1024LL); - pVnode->config.szBuf = alterReq.buffer * 1024LL * 1024LL; + req.buffer * 1024LL * 1024LL); + pVnode->config.szBuf = req.buffer * 1024LL * 1024LL; } - if (pVnode->config.szCache != alterReq.pages) { - if (metaAlterCache(pVnode->pMeta, alterReq.pages) < 0) { + if (pVnode->config.szCache != req.pages) { + if (metaAlterCache(pVnode->pMeta, req.pages) < 0) { vError("vgId:%d failed to change vnode pages from %d to %d failed since %s", TD_VID(pVnode), - pVnode->config.szCache, alterReq.pages, tstrerror(errno)); + pVnode->config.szCache, req.pages, tstrerror(errno)); return errno; } else { - vInfo("vgId:%d vnode pages is changed from %d to %d", TD_VID(pVnode), pVnode->config.szCache, alterReq.pages); - pVnode->config.szCache = alterReq.pages; + vInfo("vgId:%d vnode pages is changed from %d to %d", TD_VID(pVnode), pVnode->config.szCache, req.pages); + pVnode->config.szCache = req.pages; } } - if (pVnode->config.cacheLast != alterReq.cacheLast) { - pVnode->config.cacheLast = alterReq.cacheLast; + if (pVnode->config.cacheLast != req.cacheLast) { + pVnode->config.cacheLast = req.cacheLast; } - if (pVnode->config.walCfg.fsyncPeriod != alterReq.walFsyncPeriod) { - pVnode->config.walCfg.fsyncPeriod = alterReq.walFsyncPeriod; + if (pVnode->config.walCfg.fsyncPeriod != req.walFsyncPeriod) { + pVnode->config.walCfg.fsyncPeriod = req.walFsyncPeriod; walChanged = true; } - if (pVnode->config.walCfg.level != alterReq.walLevel) { - pVnode->config.walCfg.level = alterReq.walLevel; + if (pVnode->config.walCfg.level != req.walLevel) { + pVnode->config.walCfg.level = req.walLevel; walChanged = true; } - if (pVnode->config.tsdbCfg.keep0 != alterReq.daysToKeep0) { - pVnode->config.tsdbCfg.keep0 = alterReq.daysToKeep0; + if (pVnode->config.tsdbCfg.keep0 != req.daysToKeep0) { + pVnode->config.tsdbCfg.keep0 = req.daysToKeep0; if (!VND_IS_RSMA(pVnode)) { tsdbChanged = true; } } - if (pVnode->config.tsdbCfg.keep1 != alterReq.daysToKeep1) { - pVnode->config.tsdbCfg.keep1 = alterReq.daysToKeep1; + if (pVnode->config.tsdbCfg.keep1 != req.daysToKeep1) { + pVnode->config.tsdbCfg.keep1 = req.daysToKeep1; if (!VND_IS_RSMA(pVnode)) { tsdbChanged = true; } } - if (pVnode->config.tsdbCfg.keep2 != alterReq.daysToKeep2) { - pVnode->config.tsdbCfg.keep2 = alterReq.daysToKeep2; + if (pVnode->config.tsdbCfg.keep2 != req.daysToKeep2) { + pVnode->config.tsdbCfg.keep2 = req.daysToKeep2; if (!VND_IS_RSMA(pVnode)) { tsdbChanged = true; } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 97b174de1c7c70553011aae62b9dcf3e779c0efb..0c0ca9649a5962e0b1c4bd49796c25f7bc52912a 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1193,4 +1193,232 @@ SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); } +static void* ctgCloneDbVgroup(void* pSrc) { + return taosArrayDup((const SArray*)pSrc); +} + +static void ctgFreeDbVgroup(void* p) { + taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); +} + +static void* ctgCloneDbCfgInfo(void* pSrc) { + SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo)); + if (NULL == pDst) { + return NULL; + } + memcpy(pDst, pSrc, sizeof(SDbCfgInfo)); + return pDst; +} + +static void ctgFreeDbCfgInfo(void* p) { + taosMemoryFree(((SMetaRes*)p)->pRes); +} + +static void* ctgCloneDbInfo(void* pSrc) { + SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo)); + if (NULL == pDst) { + return NULL; + } + memcpy(pDst, pSrc, sizeof(SDbInfo)); + return pDst; +} + +static void ctgFreeDbInfo(void* p) { + taosMemoryFree(((SMetaRes*)p)->pRes); +} + +static void* ctgCloneTableMeta(void* pSrc) { + STableMeta* pMeta = pSrc; + int32_t size = sizeof(STableMeta) + (pMeta->tableInfo.numOfColumns + pMeta->tableInfo.numOfTags) * sizeof(SSchema); + STableMeta* pDst = taosMemoryMalloc(size); + if (NULL == pDst) { + return NULL; + } + memcpy(pDst, pSrc, size); + return pDst; +} + +static void ctgFreeTableMeta(void* p) { + taosMemoryFree(((SMetaRes*)p)->pRes); +} + +static void* ctgCloneVgroupInfo(void* pSrc) { + SVgroupInfo* pDst = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == pDst) { + return NULL; + } + memcpy(pDst, pSrc, sizeof(SVgroupInfo)); + return pDst; +} + +static void ctgFreeVgroupInfo(void* p) { + taosMemoryFree(((SMetaRes*)p)->pRes); +} + +static void* ctgCloneTableIndices(void* pSrc) { + return taosArrayDup((const SArray*)pSrc); +} + +static void ctgFreeTableIndices(void* p) { + taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); +} + +static void* ctgCloneFuncInfo(void* pSrc) { + SFuncInfo* pDst = taosMemoryMalloc(sizeof(SFuncInfo)); + if (NULL == pDst) { + return NULL; + } + memcpy(pDst, pSrc, sizeof(SFuncInfo)); + return pDst; +} + +static void ctgFreeFuncInfo(void* p) { + taosMemoryFree(((SMetaRes*)p)->pRes); +} + +static void* ctgCloneIndexInfo(void* pSrc) { + SIndexInfo* pDst = taosMemoryMalloc(sizeof(SIndexInfo)); + if (NULL == pDst) { + return NULL; + } + memcpy(pDst, pSrc, sizeof(SIndexInfo)); + return pDst; +} + +static void ctgFreeIndexInfo(void* p) { + taosMemoryFree(((SMetaRes*)p)->pRes); +} + +static void* ctgCloneUserAuth(void* pSrc) { + bool* pDst = taosMemoryMalloc(sizeof(bool)); + if (NULL == pDst) { + return NULL; + } + *pDst = *(bool*)pSrc; + return pDst; +} + +static void ctgFreeUserAuth(void* p) { + taosMemoryFree(((SMetaRes*)p)->pRes); +} + +static void* ctgCloneQnodeList(void* pSrc) { + return taosArrayDup((const SArray*)pSrc); +} +static void ctgFreeQnodeList(void* p) { + taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); +} + +static void* ctgCloneTableCfg(void* pSrc) { + STableCfg* pDst = taosMemoryMalloc(sizeof(STableCfg)); + if (NULL == pDst) { + return NULL; + } + memcpy(pDst, pSrc, sizeof(STableCfg)); + return pDst; +} + +static void ctgFreeTableCfg(void* p) { + taosMemoryFree(((SMetaRes*)p)->pRes); +} + +static void* ctgCloneDnodeList(void* pSrc) { + return taosArrayDup((const SArray*)pSrc); +} + +static void ctgFreeDnodeList(void* p) { + taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); +} + +static int32_t ctgCloneMetaDataArray(SArray* pSrc, FCopy copyFunc, SArray** pDst) { + if (NULL == pSrc) { + return TSDB_CODE_SUCCESS; + } + + int32_t size = taosArrayGetSize(pSrc); + *pDst = taosArrayInit(size, sizeof(SMetaRes)); + if (NULL == *pDst) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; i < size; ++i) { + SMetaRes* pRes = taosArrayGet(pSrc, i); + SMetaRes res = {.code = pRes->code, .pRes = copyFunc(pRes->pRes)}; + if (NULL == res.pRes) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArrayPush(*pDst, &res); + } + + return TSDB_CODE_SUCCESS; +} + +SMetaData* catalogCloneMetaData(SMetaData* pData) { + SMetaData* pRes = taosMemoryCalloc(1, sizeof(SMetaData)); + if (NULL == pRes) { + return NULL; + } + + int32_t code = ctgCloneMetaDataArray(pData->pDbVgroup, ctgCloneDbVgroup, &pRes->pDbVgroup); + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pDbCfg, ctgCloneDbCfgInfo, &pRes->pDbCfg); + } + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pDbInfo, ctgCloneDbInfo, &pRes->pDbInfo); + } + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pTableMeta, ctgCloneTableMeta, &pRes->pTableMeta); + } + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pTableHash, ctgCloneVgroupInfo, &pRes->pTableHash); + } + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pTableIndex, ctgCloneTableIndices, &pRes->pTableIndex); + } + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pUdfList, ctgCloneFuncInfo, &pRes->pUdfList); + } + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pIndex, ctgCloneIndexInfo, &pRes->pIndex); + } + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pUser, ctgCloneUserAuth, &pRes->pUser); + } + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pQnodeList, ctgCloneQnodeList, &pRes->pQnodeList); + } + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pTableCfg, ctgCloneTableCfg, &pRes->pTableCfg); + } + if (TSDB_CODE_SUCCESS == code) { + code = ctgCloneMetaDataArray(pData->pDnodeList, ctgCloneDnodeList, &pRes->pDnodeList); + } + + if (TSDB_CODE_SUCCESS != code) { + catalogFreeMetaData(pRes); + return NULL; + } + + return pRes; +} + +void catalogFreeMetaData(SMetaData* pData) { + if (NULL == pData) { + return; + } + + taosArrayDestroyEx(pData->pDbVgroup, ctgFreeDbVgroup); + taosArrayDestroyEx(pData->pDbCfg, ctgFreeDbCfgInfo); + taosArrayDestroyEx(pData->pDbInfo, ctgFreeDbInfo); + taosArrayDestroyEx(pData->pTableMeta, ctgFreeTableMeta); + taosArrayDestroyEx(pData->pTableHash, ctgFreeVgroupInfo); + taosArrayDestroyEx(pData->pTableIndex, ctgFreeTableIndices); + taosArrayDestroyEx(pData->pUdfList, ctgFreeFuncInfo); + taosArrayDestroyEx(pData->pIndex, ctgFreeIndexInfo); + taosArrayDestroyEx(pData->pUser, ctgFreeUserAuth); + taosArrayDestroyEx(pData->pQnodeList, ctgFreeQnodeList); + taosArrayDestroyEx(pData->pTableCfg, ctgFreeTableCfg); + taosArrayDestroyEx(pData->pDnodeList, ctgFreeDnodeList); + taosMemoryFreeClear(pData->pSvrVer); + taosMemoryFree(pData); +} diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 212a4aef54cfd25057a22b96c9b15a0d0b742f8d..f9e47f92fecaea8e7cbf5fa0b4cecaa2118c7971 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -767,6 +767,7 @@ typedef struct SStreamSessionAggOperatorInfo { SPhysiNode* pPhyNode; // create new child bool isFinal; bool ignoreExpiredData; + SHashObj* pGroupIdTbNameMap; } SStreamSessionAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -845,6 +846,7 @@ typedef struct SStreamStateAggOperatorInfo { void* pDelIterator; SArray* pChildren; // cache for children's result; bool ignoreExpiredData; + SHashObj* pGroupIdTbNameMap; } SStreamStateAggOperatorInfo; typedef struct SSortOperatorInfo { @@ -899,8 +901,12 @@ void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs); int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, const char* pkey); void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); + +void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf); void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); + int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2f191db74db0a3a509a021101d929ea8f8e9695d..8c1104f51988930776d8745fceb7f727289751eb 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1389,6 +1389,46 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS return 0; } +void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSDataBlock* pBlock = pbInfo->pRes; + + // set output datablock version + pBlock->info.version = pTaskInfo->version; + + blockDataCleanup(pBlock); + if (!hasRemainResults(pGroupResInfo)) { + return; + } + + // clear the existed group id + pBlock->info.groupId = 0; + ASSERT(!pbInfo->mergeResultBlock); + doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo); + if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { + SStreamStateAggOperatorInfo* pInfo = pOperator->info; + + char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t)); + if (tbname != NULL) { + memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); + } else { + pBlock->info.parTbName[0] = 0; + } + } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || + pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || + pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + + char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t)); + if (tbname != NULL) { + memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); + } else { + pBlock->info.parTbName[0] = 0; + } + } +} + void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 16d0c178e0f531613e7d65bb88e740e37a014a21..c968f827020999e177f2dba61886267789a3594a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3728,6 +3728,9 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pPhyNode = pPhyNode; pInfo->ignoreExpiredData = pSessionNode->window.igExpired; + pInfo->pGroupIdTbNameMap = + taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); + pOperator->name = "StreamSessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; pOperator->blocking = true; @@ -4351,7 +4354,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -4370,6 +4373,12 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv"); + if (pBlock->info.parTbName[0]) { + taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName, + TSDB_TABLE_NAME_LEN); + /*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/ + } + if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, START_TS_COLUMN_INDEX, @@ -4451,7 +4460,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + doBuildStreamResBlock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -4482,7 +4491,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } { - doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows > 0) { printDataBlock(pBInfo->pRes, "semi session"); return pBInfo->pRes; @@ -4523,6 +4532,12 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } printDataBlock(pBlock, "semi session recv"); + if (pBlock->info.parTbName[0]) { + taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName, + TSDB_TABLE_NAME_LEN); + /*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/ + } + if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, START_TS_COLUMN_INDEX, pSup->numOfExprs, 0, pWins); @@ -4566,7 +4581,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows > 0) { printDataBlock(pBInfo->pRes, "semi session"); return pBInfo->pRes; @@ -4610,6 +4625,10 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); } + + pInfo->pGroupIdTbNameMap = + taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); + pOperator->operatorType = pPhyNode->type; if (numOfChild > 0) { pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*)); @@ -4910,7 +4929,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->pDelRes, "single state"); return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -4972,7 +4991,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->pDelRes, "single state"); return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + doBuildStreamResBlock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); printDataBlock(pBInfo->pRes, "single state"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -5045,6 +5064,9 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pChildren = NULL; pInfo->ignoreExpiredData = pStateNode->window.igExpired; + pInfo->pGroupIdTbNameMap = + taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); + pOperator->name = "StreamStateAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; pOperator->blocking = true; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 86930268f14a19ac672a3fae8cea00227472b49a..eec20f91e4ab75c645fd0584ca7213fbd5356514 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -266,7 +266,7 @@ static int32_t getTableVgroup(SInsertParseContext* pCxt, int32_t tbNo, SName* pT return catalogGetTableHashVgroup(pBasicCtx->pCatalog, &conn, pTbName, pVg); } -static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* dbFname, bool isStb) { +static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName* name, bool isStb) { CHECK_CODE(getTableSchema(pCxt, tbNo, name, isStb, &pCxt->pTableMeta)); if (!isStb) { SVgroupInfo vg; @@ -276,12 +276,12 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName* return TSDB_CODE_SUCCESS; } -static int32_t getTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* dbFname) { - return getTableMetaImpl(pCxt, tbNo, name, dbFname, false); +static int32_t getTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) { + return getTableMetaImpl(pCxt, tbNo, name, false); } -static int32_t getSTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* dbFname) { - return getTableMetaImpl(pCxt, tbNo, name, dbFname, true); +static int32_t getSTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) { + return getTableMetaImpl(pCxt, tbNo, name, true); } static int32_t getDBCfg(SInsertParseContext* pCxt, const char* pDbFName, SDbCfgInfo* pInfo) { @@ -1178,7 +1178,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName* tNameGetFullDbName(&sname, dbFName); strcpy(pCxt->sTableName, sname.tname); - CHECK_CODE(getSTableMeta(pCxt, tbNo, &sname, dbFName)); + CHECK_CODE(getSTableMeta(pCxt, tbNo, &sname)); if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } @@ -1385,6 +1385,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB (*numOfRows)++; } pCxt->pSql = pRawSql; + + if (pDataBlock->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) { + break; + } } if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { @@ -1393,23 +1397,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB return TSDB_CODE_SUCCESS; } -static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STableDataBlocks* dataBuf) { - char filePathStr[TSDB_FILENAME_LEN] = {0}; - if (TK_NK_STRING == filePath.type) { - trimString(filePath.z, filePath.n, filePathStr, sizeof(filePathStr)); - } else { - strncpy(filePathStr, filePath.z, filePath.n); - } - TdFilePtr fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM); - if (NULL == fp) { - return TAOS_SYSTEM_ERROR(errno); - } - +static int32_t parseDataFromFileAgain(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName, + STableDataBlocks* dataBuf) { int32_t maxNumOfRows; CHECK_CODE(allocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows)); int32_t numOfRows = 0; - CHECK_CODE(parseCsvFile(pCxt, fp, dataBuf, maxNumOfRows, &numOfRows)); + CHECK_CODE(parseCsvFile(pCxt, pCxt->pComCxt->csvCxt.fp, dataBuf, maxNumOfRows, &numOfRows)); SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) { @@ -1417,12 +1411,38 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STa "too many rows in sql, total number of rows should be less than INT32_MAX"); } + if (!taosEOFFile(pCxt->pComCxt->csvCxt.fp)) { + pCxt->pComCxt->needMultiParse = true; + pCxt->pComCxt->csvCxt.tableNo = tableNo; + memcpy(&pCxt->pComCxt->csvCxt.tableName, pTableName, sizeof(SName)); + pCxt->pComCxt->csvCxt.pLastSqlPos = pCxt->pSql; + } + dataBuf->numOfTables = 1; pCxt->totalNum += numOfRows; return TSDB_CODE_SUCCESS; } +static int32_t parseDataFromFile(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName, SToken filePath, + STableDataBlocks* dataBuf) { + char filePathStr[TSDB_FILENAME_LEN] = {0}; + if (TK_NK_STRING == filePath.type) { + trimString(filePath.z, filePath.n, filePathStr, sizeof(filePathStr)); + } else { + strncpy(filePathStr, filePath.z, filePath.n); + } + pCxt->pComCxt->csvCxt.fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM); + if (NULL == pCxt->pComCxt->csvCxt.fp) { + return TAOS_SYSTEM_ERROR(errno); + } + + return parseDataFromFileAgain(pCxt, tableNo, pTableName, dataBuf); +} + static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { + if (!pCxt->pComCxt->needMultiParse) { + taosCloseFile(&pCxt->pComCxt->csvCxt.fp); + } taosMemoryFreeClear(pCxt->pTableMeta); destroyBoundColumnInfo(&pCxt->tags); tdDestroySVCreateTbReq(&pCxt->createTblReq); @@ -1481,7 +1501,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z); } - if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { + if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && + !pCxt->pComCxt->needMultiParse) { return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); } break; @@ -1536,7 +1557,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { NEXT_TOKEN(pCxt->pSql, sToken); autoCreateTbl = true; } else if (!existedUsing) { - CHECK_CODE(getTableMeta(pCxt, tbNum, &name, dbFName)); + CHECK_CODE(getTableMeta(pCxt, tbNum, &name)); if (TSDB_SUPER_TABLE == pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); } @@ -1577,17 +1598,22 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z); } - CHECK_CODE(parseDataFromFile(pCxt, sToken, dataBuf)); + CHECK_CODE(parseDataFromFile(pCxt, tbNum, &name, sToken, dataBuf)); pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT; tbNum++; - continue; + if (!pCxt->pComCxt->needMultiParse) { + continue; + } else { + parserInfo("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); + break; + } } return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); } - qDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + parserInfo("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); @@ -1612,6 +1638,26 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildOutput(pCxt); } +static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) { + STableDataBlocks* dataBuf = NULL; + CHECK_CODE(getTableMeta(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName)); + CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, &pCxt->pTableMeta->uid, sizeof(pCxt->pTableMeta->uid), + TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, + pCxt->pTableMeta, &dataBuf, NULL, &pCxt->createTblReq)); + CHECK_CODE(parseDataFromFileAgain(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName, dataBuf)); + if (taosEOFFile(pCxt->pComCxt->csvCxt.fp)) { + CHECK_CODE(parseInsertBody(pCxt)); + pCxt->pComCxt->needMultiParse = false; + return TSDB_CODE_SUCCESS; + } + parserInfo("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + // merge according to vgId + if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { + CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); + } + return buildOutput(pCxt); +} + // INSERT INTO // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] @@ -1621,7 +1667,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache) { SInsertParseContext context = { .pComCxt = pContext, - .pSql = (char*)pContext->pSql, + .pSql = pContext->needMultiParse ? (char*)pContext->csvCxt.pLastSqlPos : (char*)pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, .pTableMeta = NULL, .createTblReq = {0}, @@ -1691,10 +1737,16 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache context.pOutput->payloadType = PAYLOAD_TYPE_KV; - int32_t code = skipInsertInto(&context.pSql, &context.msg); - if (TSDB_CODE_SUCCESS == code) { - code = parseInsertBody(&context); + int32_t code = TSDB_CODE_SUCCESS; + if (!context.pComCxt->needMultiParse) { + code = skipInsertInto(&context.pSql, &context.msg); + if (TSDB_CODE_SUCCESS == code) { + code = parseInsertBody(&context); + } + } else { + code = parseInsertBodyAgain(&context); } + if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) { SName* pTable = taosHashIterate(context.pTableNameHashObj, NULL); while (NULL != pTable) { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 2fe6ebfb79447653731ff907e5128dfededdf111..748478778adf2a16434c7290b213a21fab8b1ea2 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -214,6 +214,16 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata return code; } +void qDestroyParseContext(SParseContext* pCxt) { + if (NULL == pCxt) { + return; + } + + taosArrayDestroy(pCxt->pTableMetaPos); + taosArrayDestroy(pCxt->pTableVgroupPos); + taosMemoryFree(pCxt); +} + void qDestroyQuery(SQuery* pQueryNode) { nodesDestroyNode((SNode*)pQueryNode); } int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index fcaa5af05ccdb3a9fe1cb8a51177cab679e0dc84..a9360f796c9eee31c45007ebdc365aa2455365fd 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -130,16 +130,16 @@ void generatePerformanceSchema(MockCatalogService* mcs) { * c5 | column | DOUBLE | 8 | */ void generateTestTables(MockCatalogService* mcs, const std::string& db) { - ITableBuilder& builder = mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6) - .setPrecision(TSDB_TIME_PRECISION_MILLI) - .setVgid(1) - .addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) - .addColumn("c1", TSDB_DATA_TYPE_INT) - .addColumn("c2", TSDB_DATA_TYPE_BINARY, 20) - .addColumn("c3", TSDB_DATA_TYPE_BIGINT) - .addColumn("c4", TSDB_DATA_TYPE_DOUBLE) - .addColumn("c5", TSDB_DATA_TYPE_DOUBLE); - builder.done(); + mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6) + .setPrecision(TSDB_TIME_PRECISION_MILLI) + .setVgid(1) + .addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) + .addColumn("c1", TSDB_DATA_TYPE_INT) + .addColumn("c2", TSDB_DATA_TYPE_BINARY, 20) + .addColumn("c3", TSDB_DATA_TYPE_BIGINT) + .addColumn("c4", TSDB_DATA_TYPE_DOUBLE) + .addColumn("c5", TSDB_DATA_TYPE_DOUBLE) + .done(); } /* diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index 14c991917bf534fb2719b2c9d90aff9325042615..bf27fd2e13191cbdae8bdefc478e5f8f0bc86375 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -343,7 +343,6 @@ class ParserTestBaseImpl { unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); doParseSql(&cxt, query.get()); - SQuery* pQuery = *(query.get()); if (g_dump) { dump(); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ac063cb50dcbb81a2fec19728444b9b76c69fa70..dbb5214415624c4215e2bfe33a7ed5aa3fd74729 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1939,7 +1939,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; fi->data = taosMemoryCalloc(1, bytes); - } else{ + } else { if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE /* fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 9cba94d85a1ffbfe2f2f1634589039e3a96b7585..b4b99be6362c31ac752df190920f55b22a6aec32 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -363,7 +363,11 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index); - if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { + if (NULL == block) { + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (ref->slotId >= taosArrayGetSize(block->pDataBlock)) { sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 4ead1147e47e8777a422cd8f63a74d4017a368a5..91044a96b53aed1c8d32c8fc57278f9b14499fd5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -709,8 +709,15 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int16_t outputType = GET_PARAM_TYPE(&pOutput[0]); int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]); - char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows + 1, 1); - char *output = outputBuf; + int32_t code = TSDB_CODE_SUCCESS; + char * convBuf = taosMemoryMalloc(inputLen); + char * output = taosMemoryCalloc(1, outputLen + TSDB_NCHAR_SIZE); + char buf[400] = {0}; + + if (convBuf == NULL || output == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { @@ -723,17 +730,18 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp switch(outputType) { case TSDB_DATA_TYPE_TINYINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(int8_t *)output = taosStr2Int8(varDataVal(input), NULL, 10); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(int8_t *)output = taosStr2Int8(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(int8_t *)output = taosStr2Int8(newBuf, NULL, 10); - taosMemoryFree(newBuf); + + convBuf[len] = 0; + *(int8_t *)output = taosStr2Int8(convBuf, NULL, 10); } else { GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input); } @@ -741,17 +749,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_SMALLINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(int16_t *)output = taosStr2Int16(varDataVal(input), NULL, 10); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(int16_t *)output = taosStr2Int16(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(int16_t *)output = taosStr2Int16(newBuf, NULL, 10); - taosMemoryFree(newBuf); + convBuf[len] = 0; + *(int16_t *)output = taosStr2Int16(convBuf, NULL, 10); } else { GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input); } @@ -759,17 +767,18 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_INT: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(int32_t *)output = taosStr2Int32(varDataVal(input), NULL, 10); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(int32_t *)output = taosStr2Int32(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(int32_t *)output = taosStr2Int32(newBuf, NULL, 10); - taosMemoryFree(newBuf); + + convBuf[len] = 0; + *(int32_t *)output = taosStr2Int32(convBuf, NULL, 10); } else { GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input); } @@ -777,17 +786,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_BIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(int64_t *)output = taosStr2Int64(varDataVal(input), NULL, 10); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(int64_t *)output = taosStr2Int64(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(int64_t *)output = taosStr2Int64(newBuf, NULL, 10); - taosMemoryFree(newBuf); + convBuf[len] = 0; + *(int64_t *)output = taosStr2Int64(convBuf, NULL, 10); } else { GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); } @@ -795,17 +804,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_UTINYINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(uint8_t *)output = taosStr2UInt8(varDataVal(input), NULL, 10); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(uint8_t *)output = taosStr2UInt8(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(uint8_t *)output = taosStr2UInt8(newBuf, NULL, 10); - taosMemoryFree(newBuf); + convBuf[len] = 0; + *(uint8_t *)output = taosStr2UInt8(convBuf, NULL, 10); } else { GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input); } @@ -813,17 +822,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_USMALLINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(uint16_t *)output = taosStr2UInt16(varDataVal(input), NULL, 10); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(uint16_t *)output = taosStr2UInt16(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(uint16_t *)output = taosStr2UInt16(newBuf, NULL, 10); - taosMemoryFree(newBuf); + convBuf[len] = 0; + *(uint16_t *)output = taosStr2UInt16(convBuf, NULL, 10); } else { GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input); } @@ -831,17 +840,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_UINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(uint32_t *)output = taosStr2UInt32(varDataVal(input), NULL, 10); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(uint32_t *)output = taosStr2UInt32(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(uint32_t *)output = taosStr2UInt32(newBuf, NULL, 10); - taosMemoryFree(newBuf); + convBuf[len] = 0; + *(uint32_t *)output = taosStr2UInt32(convBuf, NULL, 10); } else { GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input); } @@ -849,17 +858,18 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_UBIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(uint64_t *)output = taosStr2UInt64(varDataVal(input), NULL, 10); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(uint64_t *)output = taosStr2UInt64(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(uint64_t *)output = taosStr2UInt64(newBuf, NULL, 10); - taosMemoryFree(newBuf); + + convBuf[len] = 0; + *(uint64_t *)output = taosStr2UInt64(convBuf, NULL, 10); } else { GET_TYPED_DATA(*(uint64_t *)output, uint64_t, inputType, input); } @@ -867,17 +877,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_FLOAT: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(float *)output = taosStr2Float(varDataVal(input), NULL); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(float *)output = taosStr2Float(buf, NULL); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(float *)output = taosStr2Float(newBuf, NULL); - taosMemoryFree(newBuf); + convBuf[len] = 0; + *(float *)output = taosStr2Float(convBuf, NULL); } else { GET_TYPED_DATA(*(float *)output, float, inputType, input); } @@ -885,17 +895,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_DOUBLE: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(double *)output = taosStr2Double(varDataVal(input), NULL); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(double *)output = taosStr2Double(buf, NULL); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(double *)output = taosStr2Double(newBuf, NULL); - taosMemoryFree(newBuf); + convBuf[len] = 0; + *(double *)output = taosStr2Double(convBuf, NULL); } else { GET_TYPED_DATA(*(double *)output, double, inputType, input); } @@ -903,17 +913,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_BOOL: { if (inputType == TSDB_DATA_TYPE_BINARY) { - *(bool *)output = taosStr2Int8(varDataVal(input), NULL, 10); + memcpy(buf, varDataVal(input), varDataLen(input)); + buf[varDataLen(input)] = 0; + *(bool *)output = taosStr2Int8(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } - newBuf[len] = 0; - *(bool *)output = taosStr2Int8(newBuf, NULL, 10); - taosMemoryFree(newBuf); + convBuf[len] = 0; + *(bool *)output = taosStr2Int8(convBuf, NULL, 10); } else { GET_TYPED_DATA(*(bool *)output, bool, inputType, input); } @@ -937,29 +947,27 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_BINARY: { if (inputType == TSDB_DATA_TYPE_BOOL) { + // NOTE: sprintf will append '\0' at the end of string int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), *(int8_t *)input ? "true" : "false"); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_BINARY) { int32_t len = TMIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE); - len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input)); + memcpy(varDataVal(output), varDataVal(input), len); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - char *newBuf = taosMemoryCalloc(1, inputLen); - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); if (len < 0) { - taosMemoryFree(newBuf); - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } len = TMIN(len, outputLen - VARSTR_HEADER_SIZE); - memcpy(varDataVal(output), newBuf, len); + memcpy(varDataVal(output), convBuf, len); varDataSetLen(output, len); - taosMemoryFree(newBuf); } else { - char tmp[400] = {0}; - NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); - int32_t len = (int32_t)strlen(tmp); + NUM_TO_STRING(inputType, input, sizeof(buf), buf); + int32_t len = (int32_t)strlen(buf); len = (outputLen - VARSTR_HEADER_SIZE) > len ? len : (outputLen - VARSTR_HEADER_SIZE); - memcpy(varDataVal(output), tmp, len); + memcpy(varDataVal(output), buf, len); varDataSetLen(output, len); } break; @@ -972,14 +980,17 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false" ); bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } + varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_BINARY) { len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen; bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { @@ -987,38 +998,39 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp memcpy(output, input, len + VARSTR_HEADER_SIZE); varDataSetLen(output, len); } else { - char tmp[400] = {0}; - NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); - len = (int32_t)strlen(tmp); + NUM_TO_STRING(inputType, input, sizeof(buf), buf); + len = (int32_t)strlen(buf); len = outputCharLen > len ? len : outputCharLen; - bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); + bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } varDataSetLen(output, len); } + //for constant conversion, need to set proper length of pOutput description if (len < outputLen) { pOutput->columnData->info.bytes = len + VARSTR_HEADER_SIZE; } + break; } default: { - return TSDB_CODE_FAILED; + code = TSDB_CODE_FAILED; + goto _end; } } colDataAppend(pOutput->columnData, i, output, false); - if (IS_VAR_DATA_TYPE(outputType)) { - output += varDataTLen(output); - } else { - output += tDataTypes[outputType].bytes; - } } pOutput->numOfRows = pInput->numOfRows; - taosMemoryFree(outputBuf); - return TSDB_CODE_SUCCESS; + + _end: + taosMemoryFree(output); + taosMemoryFree(convBuf); + return code; } int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { @@ -1400,8 +1412,6 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { /* unix timestamp or ts column*/ GET_TYPED_DATA(timeVal[k], int64_t, type, input[k]); if (type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : - (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); int64_t timeValSec = timeVal[k] / factor; if (timeValSec < 1000000000) { timeVal[k] = timeValSec; @@ -1992,7 +2002,7 @@ int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * } } - if (hasNull) { + if (hasNull || (count == 0)) { colDataAppendNULL(pOutputData, 0); } else { if (IS_SIGNED_NUMERIC_TYPE(type)) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index fe2a970aaa811cc02f589c845e3c62b7e70af8e8..339b18bc06e20f5e42b75927c01429dbc32b06b3 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -281,7 +281,7 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r if (taosParseTime(buf, &value, strlen(buf), pOut->columnData->info.precision, tsDaylight) != TSDB_CODE_SUCCESS) { value = 0; } - + colDataAppendInt64(pOut->columnData, rowIndex, &value); } @@ -300,29 +300,29 @@ static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowI *overflow = 0; } } - + switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_TINYINT: { int8_t value = (int8_t)taosStr2Int8(buf, NULL, 10); - + colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; - } + } case TSDB_DATA_TYPE_SMALLINT: { int16_t value = (int16_t)taosStr2Int16(buf, NULL, 10); colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); break; - } + } case TSDB_DATA_TYPE_INT: { int32_t value = (int32_t)taosStr2Int32(buf, NULL, 10); colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); break; - } + } case TSDB_DATA_TYPE_BIGINT: { int64_t value = (int64_t)taosStr2Int64(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); break; - } + } } } @@ -347,22 +347,22 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t ro uint8_t value = (uint8_t)taosStr2UInt8(buf, NULL, 10); colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; - } + } case TSDB_DATA_TYPE_USMALLINT: { uint16_t value = (uint16_t)taosStr2UInt16(buf, NULL, 10); colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); break; - } + } case TSDB_DATA_TYPE_UINT: { uint32_t value = (uint32_t)taosStr2UInt32(buf, NULL, 10); colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); break; - } + } case TSDB_DATA_TYPE_UBIGINT: { uint64_t value = (uint64_t)taosStr2UInt64(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); break; - } + } } } @@ -372,7 +372,7 @@ static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIn colDataAppendFloat(pOut->columnData, rowIndex, &value); return; } - + double value = taosStr2Double(buf, NULL); colDataAppendDouble(pOut->columnData, rowIndex, &value); } @@ -488,11 +488,11 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in tmp[len] = 0; } } - + (*func)(tmp, pOut, i, overflow); taosMemoryFreeClear(tmp); } - + return TSDB_CODE_SUCCESS; } @@ -529,24 +529,24 @@ void* ncharTobinary(void *buf){ // todo need to remove , if tobinary bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft, bool *freeRight){ - if(optr == OP_TYPE_JSON_CONTAINS) { + if (optr == OP_TYPE_JSON_CONTAINS) { return true; } - if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){ + if (typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON) { return true; } - if(typeLeft == TSDB_DATA_TYPE_JSON){ - if(tTagIsJson(*pLeftData)){ + if (typeLeft == TSDB_DATA_TYPE_JSON) { + if (tTagIsJson(*pLeftData)) { terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR; return false; } typeLeft = **pLeftData; (*pLeftData) ++; } - if(typeRight == TSDB_DATA_TYPE_JSON){ - if(tTagIsJson(*pLeftData)){ + if (typeRight == TSDB_DATA_TYPE_JSON) { + if (tTagIsJson(*pLeftData)) { terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR; return false; } @@ -554,71 +554,71 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t (*pRightData) ++; } - if(optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH){ - if(typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY){ + if (optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH) { + if (typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY) { return false; } } // if types can not comparable - if((IS_NUMERIC_TYPE(typeLeft) && !IS_NUMERIC_TYPE(typeRight)) || - (IS_NUMERIC_TYPE(typeRight) && !IS_NUMERIC_TYPE(typeLeft)) || - (IS_VAR_DATA_TYPE(typeLeft) && !IS_VAR_DATA_TYPE(typeRight)) || - (IS_VAR_DATA_TYPE(typeRight) && !IS_VAR_DATA_TYPE(typeLeft)) || - ((typeLeft == TSDB_DATA_TYPE_BOOL) && (typeRight != TSDB_DATA_TYPE_BOOL)) || - ((typeRight == TSDB_DATA_TYPE_BOOL) && (typeLeft != TSDB_DATA_TYPE_BOOL))) + if ((IS_NUMERIC_TYPE(typeLeft) && !IS_NUMERIC_TYPE(typeRight)) || + (IS_NUMERIC_TYPE(typeRight) && !IS_NUMERIC_TYPE(typeLeft)) || + (IS_VAR_DATA_TYPE(typeLeft) && !IS_VAR_DATA_TYPE(typeRight)) || + (IS_VAR_DATA_TYPE(typeRight) && !IS_VAR_DATA_TYPE(typeLeft)) || + ((typeLeft == TSDB_DATA_TYPE_BOOL) && (typeRight != TSDB_DATA_TYPE_BOOL)) || + ((typeRight == TSDB_DATA_TYPE_BOOL) && (typeLeft != TSDB_DATA_TYPE_BOOL))) return false; - if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){ + if (typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL) { *isNull = true; return true; } int8_t type = vectorGetConvertType(typeLeft, typeRight); - if(type == 0) { + if (type == 0) { *fp = filterGetCompFunc(typeLeft, optr); return true; } *fp = filterGetCompFunc(type, optr); - if(IS_NUMERIC_TYPE(type)){ - if(typeLeft == TSDB_DATA_TYPE_NCHAR) { + if (IS_NUMERIC_TYPE(type)) { + if (typeLeft == TSDB_DATA_TYPE_NCHAR) { ASSERT(0); // convertNcharToDouble(*pLeftData, pLeftOut); // *pLeftData = pLeftOut; - } else if(typeLeft == TSDB_DATA_TYPE_BINARY) { + } else if (typeLeft == TSDB_DATA_TYPE_BINARY) { ASSERT(0); // convertBinaryToDouble(*pLeftData, pLeftOut); // *pLeftData = pLeftOut; - } else if(typeLeft != type) { + } else if (typeLeft != type) { convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); *pLeftData = pLeftOut; } - if(typeRight == TSDB_DATA_TYPE_NCHAR) { + if (typeRight == TSDB_DATA_TYPE_NCHAR) { ASSERT(0); // convertNcharToDouble(*pRightData, pRightOut); // *pRightData = pRightOut; - } else if(typeRight == TSDB_DATA_TYPE_BINARY) { + } else if (typeRight == TSDB_DATA_TYPE_BINARY) { ASSERT(0); // convertBinaryToDouble(*pRightData, pRightOut); // *pRightData = pRightOut; - } else if(typeRight != type) { + } else if (typeRight != type) { convertNumberToNumber(*pRightData, pRightOut, typeRight, type); *pRightData = pRightOut; } - }else if(type == TSDB_DATA_TYPE_BINARY){ - if(typeLeft == TSDB_DATA_TYPE_NCHAR){ + } else if (type == TSDB_DATA_TYPE_BINARY) { + if (typeLeft == TSDB_DATA_TYPE_NCHAR) { *pLeftData = ncharTobinary(*pLeftData); *freeLeft = true; } - if(typeRight == TSDB_DATA_TYPE_NCHAR){ + if (typeRight == TSDB_DATA_TYPE_NCHAR) { *pRightData = ncharTobinary(*pRightData); *freeRight = true; } - }else{ + } else { ASSERT(0); } @@ -636,7 +636,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1 colDataAppendNULL(pOutputCol, i); continue; } - + int64_t value = 0; GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value); @@ -653,7 +653,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1 colDataAppendNULL(pOutputCol, i); continue; } - + uint64_t value = 0; GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value); @@ -670,7 +670,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1 colDataAppendNULL(pOutputCol, i); continue; } - + double value = 0; GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%lf", value); @@ -698,7 +698,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* sclError("input column is NULL, hashFilter %p", pIn->pHashFilter); return TSDB_CODE_APP_ERROR; } - + int16_t inType = pInputCol->info.type; int16_t outType = pOutputCol->info.type; @@ -710,14 +710,14 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* ASSERT(1 == pIn->numOfRows); pOut->numOfRows = 0; - + if (IS_SIGNED_NUMERIC_TYPE(outType)) { int64_t minValue = tDataTypes[outType].minValue; int64_t maxValue = tDataTypes[outType].maxValue; - + double value = 0; GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0)); - + if (value > maxValue) { *overflow = 1; return TSDB_CODE_SUCCESS; @@ -730,10 +730,10 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { uint64_t minValue = (uint64_t)tDataTypes[outType].minValue; uint64_t maxValue = (uint64_t)tDataTypes[outType].maxValue; - + double value = 0; GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0)); - + if (value > maxValue) { *overflow = 1; return TSDB_CODE_SUCCESS; @@ -820,7 +820,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* colDataAppendNULL(pOutputCol, i); continue; } - + uint8_t value = 0; GET_TYPED_DATA(value, uint8_t, inType, colDataGetData(pInputCol, i)); colDataAppendInt8(pOutputCol, i, (int8_t *)&value); @@ -833,7 +833,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* colDataAppendNULL(pOutputCol, i); continue; } - + uint16_t value = 0; GET_TYPED_DATA(value, uint16_t, inType, colDataGetData(pInputCol, i)); colDataAppendInt16(pOutputCol, i, (int16_t *)&value); @@ -846,7 +846,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* colDataAppendNULL(pOutputCol, i); continue; } - + uint32_t value = 0; GET_TYPED_DATA(value, uint32_t, inType, colDataGetData(pInputCol, i)); colDataAppendInt32(pOutputCol, i, (int32_t *)&value); @@ -859,7 +859,7 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* colDataAppendNULL(pOutputCol, i); continue; } - + uint64_t value = 0; GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); colDataAppendInt64(pOutputCol, i, (int64_t*)&value); @@ -872,12 +872,12 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* colDataAppendNULL(pOutputCol, i); continue; } - + float value = 0; GET_TYPED_DATA(value, float, inType, colDataGetData(pInputCol, i)); colDataAppendFloat(pOutputCol, i, (float*)&value); } - break; + break; } case TSDB_DATA_TYPE_DOUBLE: { for (int32_t i = 0; i < pIn->numOfRows; ++i) { @@ -885,14 +885,14 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* colDataAppendNULL(pOutputCol, i); continue; } - + double value = 0; GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); colDataAppendDouble(pOutputCol, i, (double*)&value); } - break; + break; } - case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { return vectorConvertToVarData(pIn, pOut, inType, outType); } @@ -966,10 +966,10 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p return TSDB_CODE_SUCCESS; } - SScalarParam *param1 = NULL, *paramOut1 = NULL; + SScalarParam *param1 = NULL, *paramOut1 = NULL; SScalarParam *param2 = NULL, *paramOut2 = NULL; int32_t code = 0; - + if (leftType < rightType) { param1 = pLeft; param2 = pRight; @@ -993,7 +993,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p return code; } } - + if (type != GET_PARAM_TYPE(param2)) { code = vectorConvertScalarParam(param2, paramOut2, type); if (code) { @@ -1136,7 +1136,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut } *output = getVectorBigintValueFnLeft(pLeftCol->pData, i) + getVectorBigintValueFnRight(pRightCol->pData, i); } - } + } } else { double *output = (double *)pOutputCol->pData; _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); @@ -1198,7 +1198,7 @@ static void vectorMathTsSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pR } *output = taosTimeAdd(getVectorBigintValueFnLeft(pLeftCol->pData, i), -getVectorBigintValueFnRight(pRightCol->pData, 0), pRightCol->info.scale, pRightCol->info.precision); - + } } } @@ -1706,7 +1706,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * int32_t lType = GET_PARAM_TYPE(pLeft); int32_t rType = GET_PARAM_TYPE(pRight); __compar_fn_t fp = NULL; - + if (lType == rType) { fp = filterGetCompFunc(lType, optr); } else { @@ -1736,9 +1736,9 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * } void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { - SScalarParam pLeftOut = {0}; + SScalarParam pLeftOut = {0}; SScalarParam pRightOut = {0}; - SScalarParam *param1 = NULL; + SScalarParam *param1 = NULL; SScalarParam *param2 = NULL; if (SCL_NO_NEED_CONVERT_COMPARISION(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) { @@ -1762,7 +1762,7 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut vectorCompareImpl(param1, param2, pOut, _ord, optr); sclFreeParam(&pLeftOut); - sclFreeParam(&pRightOut); + sclFreeParam(&pRightOut); } void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f663f4363d7afe7e0c258eadb94554678e2ce0e3..a148c1e36bbdabb249f6dc64dc7c86c6934e641e 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1485,7 +1485,7 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * if (dropOfp) { int ret = 0; SPgno pgno = *(SPgno *)(pCell + nLocal - sizeof(SPgno)); - int nLeft = nPayload - nLocal + sizeof(SPgno); + int nLeft = nPayload - nLocal + sizeof(SPgno) + nHeader; SPage *ofp; int bytes; diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp index 58ea6147ef0ec9e76513a0b3c66513bc9321053d..d98c271edb128d3ba4c8b465dcd0aeba287bd0d4 100644 --- a/source/libs/tdb/test/tdbExOVFLTest.cpp +++ b/source/libs/tdb/test/tdbExOVFLTest.cpp @@ -238,8 +238,8 @@ TEST(TdbOVFLPagesTest, TbGetTest) { } } -TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) { - // TEST(TdbOVFLPagesTest, TbDeleteTest) { +// TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) { +TEST(TdbOVFLPagesTest, TbDeleteTest) { int ret = 0; taosRemoveDir("tdb"); @@ -267,7 +267,8 @@ TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) { tdbBegin(pEnv, &txn); // generate value payload - char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[((4083 - 4 - 3 - 2) + 1) * 2]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) int valLen = sizeof(val) / sizeof(val[0]); generateBigVal(val, valLen); @@ -340,8 +341,8 @@ tdbBegin(pEnv, &txn); tdbTxnClose(&txn); } -TEST(tdb_test, DISABLED_simple_insert1) { - // TEST(tdb_test, simple_insert1) { +// TEST(tdb_test, DISABLED_simple_insert1) { +TEST(tdb_test, simple_insert1) { int ret; TDB *pEnv; TTB *pDb; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 8f97f78556916b43bf9a9dc5c29e8542c051d6da..233d6a87b86fee566763fdd1c8aec9262008ab42 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -150,7 +150,6 @@ int walCheckAndRepairMeta(SWal* pWal) { const char* idxPattern = "^[0-9]+.idx$"; regex_t logRegPattern; regex_t idxRegPattern; - SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo)); regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); @@ -163,6 +162,8 @@ int walCheckAndRepairMeta(SWal* pWal) { return -1; } + SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo)); + // scan log files and build new meta TdDirEntryPtr pDirEntry; while ((pDirEntry = taosReadDir(pDir)) != NULL) { diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 2a9d0c8535b6cc8ccde0a75013d0fbc7187e7d41..b7cb20896bb79d5cee6eba6fed74dc96d87173f6 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -52,3 +52,7 @@ else() os PUBLIC dl m rt ) endif() + +IF (JEMALLOC_ENABLED) + target_link_libraries(os PUBLIC -ljemalloc) +ENDIF () \ No newline at end of file diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index c1e02b48aa047e1043a41820ed77b6626e090aa2..bfeef248cdce61bba62d8cc5dcfd175b09559448 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -267,21 +267,29 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void return -1; } #ifdef WINDOWS +#ifdef TCP_KEEPCNT if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } +#endif +#ifdef TCP_KEEPIDLE if (level == SOL_TCP && optname == TCP_KEEPIDLE) { return 0; } +#endif +#ifdef TCP_KEEPINTVL if (level == SOL_TCP && optname == TCP_KEEPINTVL) { return 0; } +#endif +#ifdef TCP_KEEPCNT if (level == SOL_TCP && optname == TCP_KEEPCNT) { return 0; } +#endif return setsockopt(pSocket->fd, level, optname, optval, optlen); #else @@ -601,26 +609,32 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { #ifndef __APPLE__ // all fails on macosx +#ifdef TCP_KEEPCNT int32_t probes = 3; if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { // printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(&pSocket); return -1; } +#endif +#ifdef TCP_KEEPIDLE int32_t alivetime = 10; if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { // printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(&pSocket); return -1; } +#endif +#ifdef TCP_KEEPINTVL int32_t interval = 3; if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { // printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(&pSocket); return -1; } +#endif #endif // __APPLE__ int32_t nodelay = 1; diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py index 371d147efc38735b4b0efdc19dc9288edbca5030..9a94632a12ea7746cc2d2ff9a660f8c83dbdb9c6 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py @@ -132,7 +132,7 @@ class TDTestCase: tdSql.error("create mnode on dnode 2") tdSql.query("select * from information_schema.ins_dnodes;") print(tdSql.queryResult) - clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkDnodes(dnodeNumbers, 60) # create database and stable clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index c37e3541d4e7b70d484fd17030b9ce581993bb59..28c481083395c74c02238d380946bd00c0b74c0f 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -37,10 +37,10 @@ class ClusterComCheck: tdSql.init(conn.cursor()) # tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkDnodes(self,dnodeNumbers): + def checkDnodes(self,dnodeNumbers, timeout=30): count=0 # print(tdSql) - while count < 30: + while count < timeout: tdSql.query("select * from information_schema.ins_dnodes") # tdLog.debug(tdSql.queryResult) status=0 @@ -50,14 +50,14 @@ class ClusterComCheck: tdLog.info(status) if status == dnodeNumbers: - tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within 30s! " %dnodeNumbers) + tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within %ds! " % (dnodeNumbers, count)) return True count+=1 time.sleep(1) else: tdSql.query("select * from information_schema.ins_dnodes") tdLog.debug(tdSql.queryResult) - tdLog.exit("it find cluster with %d dnodes but check that there dnodes are not ready within 30s ! "%dnodeNumbers) + tdLog.exit("it find cluster with %d dnodes but check that there dnodes are not ready within %ds ! "% (dnodeNumbers, timeout)) def checkDbRows(self,dbNumbers): dbNumbers=int(dbNumbers) diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 45a6f8c16f60dc2d4cc57f4e57a9e44034ec3aaa..552b77e6e9db1825de474c3398a01dd983a8aed1 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -28,7 +28,4 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -IF (TD_LINUX_64 AND JEMALLOC_ENABLED) - add_dependencies(taosd jemalloc) -ENDIF () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 506897cf5b4ec1fecfd937519e2261092999091c..8aafd747afe2f5972e0010d0be0222a7d56e961b 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -1133,6 +1133,10 @@ bool firstMatchCommand(TAOS* con, SShellCmd* cmd) { // print to screen printScreen(con, cmd, match); +#ifdef WINDOWS + printf("\r"); + shellShowOnScreen(cmd); +#endif freeCommand(input); taosMemoryFree(input); return true; @@ -1195,6 +1199,10 @@ bool nextMatchCommand(TAOS* con, SShellCmd* cmd, SWords* firstMatch) { // print to screen printScreen(con, cmd, match); +#ifdef WINDOWS + printf("\r"); + shellShowOnScreen(cmd); +#endif // free if (input->source) { diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index cffa02824e28dcbd84e7930b9ae7df6422f38610..0b26d685fdcd4ab6bf08683494d2bf7ca57858d1 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -417,8 +417,9 @@ char taosGetConsoleChar() { static char mbStr[5]; static unsigned long bufLen = 0; static uint16_t bufIndex = 0, mbStrIndex = 0, mbStrLen = 0; + CONSOLE_READCONSOLE_CONTROL inputControl={ sizeof(CONSOLE_READCONSOLE_CONTROL), 0, 1< 0 && buf[0] == 0) bufLen = 0; bufIndex = 0; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index abb6cf84d553654a16aa5a32dd2f0335b1a6d23b..fd0ea6032357db68fc523a61f56de6aa0df54017 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1112,7 +1112,9 @@ int32_t shellExecute() { #ifdef WEBSOCKET if (!shell.args.restful && !shell.args.cloud) { #endif +#ifndef WINDOWS printfIntroduction(); +#endif shellGetGrantInfo(); #ifdef WEBSOCKET }