diff --git a/README-CN.md b/README-CN.md index 627b35698c21cf234b011c17c85c7faee53671b3..437d671bb97a1009f5d48d2423a3d17f75bf1494 100644 --- a/README-CN.md +++ b/README-CN.md @@ -107,7 +107,7 @@ sudo yum config-manager --set-enabled Powertools ### macOS ``` -sudo brew install argp-standalone pkgconfig +brew install argp-standalone pkgconfig ``` ### 设置 golang 开发环境 @@ -276,7 +276,7 @@ sudo make install 安装成功后，可以在应用程序中双击 TDengine 图标启动服务，或者在终端中启动 TDengine 服务： ```bash -launchctl start taosd +launchctl start com.tdengine.taosd ``` 用户可以使用 TDengine CLI 来连接 TDengine 服务，在终端中，输入： diff --git a/README.md b/README.md index 00d36efb5a630a44eeeaa9b3b47df5c649a0a305..8d2567a816f9048cd689afa06426e9c88979a354 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ sudo yum config-manager --set-enabled powertools ### macOS ``` -sudo brew install argp-standalone pkgconfig +brew install argp-standalone pkgconfig ``` ### Setup golang environment @@ -280,7 +280,7 @@ Installing from source code will also configure service management for TDengine. To start the service after installation, double-click the /applications/TDengine to start the program, or in a terminal, use: ```bash -launchctl start taosd +launchctl start com.tdengine.taosd ``` Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: diff --git a/cmake/cmake.platform b/cmake/cmake.platform index e4d440d76edbfcf0d1d6f932cfa598fe0a0f43d2..3e239d2e0c9f1fb53a4c156cab52801f6206df75 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -45,10 +45,19 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare") MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.") - IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64") - MESSAGE("Current system arch is 64") + IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") + MESSAGE("Current system arch is arm64") SET(TD_DARWIN_64 TRUE) + SET(TD_DARWIN_ARM64 TRUE) ADD_DEFINITIONS("-D_TD_DARWIN_64") + ADD_DEFINITIONS("-D_TD_DARWIN_ARM64") + ENDIF () + IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64") + MESSAGE("Current system arch is x86_64") + SET(TD_DARWIN_64 TRUE) + SET(TD_DARWIN_X64 TRUE) + ADD_DEFINITIONS("-D_TD_DARWIN_64") + ADD_DEFINITIONS("-D_TD_DARWIN_X64") ENDIF () ADD_DEFINITIONS("-DHAVE_UNISTD_H") diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 7bde332c8c0f068017aa15e5a1172716b78698bb..73409a950a848662afc87a14b9f4c96dec4a625d 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG d58230c + GIT_TAG 4d02980 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a1a6cd4519294fa99a5bee41ae21aad4c1731373..969724cc8d0ebd68f3a3cbdf5716bdcb4efb5ff9 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -270,7 +270,7 @@ if(${JEMALLOC_ENABLED}) PREFIX "jemalloc" SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls + CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ --disable-initial-exec-tls --with-malloc-conf='background_thread:true,metadata_thp:auto' BUILD_COMMAND ${MAKE} ) INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include) diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index f734e354fb16c8ca7342f04fc46f869009bbd8c8..7800fb1c88a59c28945a196e1e5dac95427bf1f6 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -189,13 +189,13 @@ After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengin -After the installation is complete, double-click the /applications/TDengine to start the program, or run `launchctl start taosd` to start TDengine Server. +After the installation is complete, double-click the /applications/TDengine to start the program, or run `launchctl start com.tdengine.taosd` to start TDengine Server. The following `launchctl` commands can help you manage TDengine service: -- Start TDengine Server: `launchctl start taosd` +- Start TDengine Server: `launchctl start com.tdengine.taosd` -- Stop TDengine Server: `launchctl stop taosd` +- Stop TDengine Server: `launchctl stop com.tdengine.taosd` - Check TDengine Server status: `launchctl list | grep taosd` diff --git a/docs/en/12-taos-sql/28-recovery.md b/docs/en/12-taos-sql/28-recovery.md index 14ac14f8673fba05fee09317de927df00effed0f..e869ffc45ff7420836d91a2536d32ce6b08a0d82 100644 --- a/docs/en/12-taos-sql/28-recovery.md +++ b/docs/en/12-taos-sql/28-recovery.md @@ -16,10 +16,10 @@ You can use the SHOW CONNECTIONS statement to find the conn_id. ## Terminate a Query ```sql -SHOW QUERY query_id; +KILL QUERY kill_id; ``` -You can use the SHOW QUERIES statement to find the query_id. +You can use the SHOW QUERIES statement to find the kill_id. ## Terminate a Transaction diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index d95086d4c498bf61564eb953ea54da93a8bac404..6295d8553d54033ee08413696d45503416394f27 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -231,7 +231,7 @@ The parameters related to database creation are configured in `dbinfo` in the js - **name**: specify the name of the database. -- **drop**: indicate whether to delete the database before inserting. The default is true. +- **drop**: indicate whether to delete the database before inserting. The value can be 'yes' or 'no'. No means do not drop. The default is to drop. #### Stream processing related configuration parameters diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index d91675a0ab7209bdae9b4aa5a80e984c7605f2f9..8053c4d25d527adf3e2c22fd107c7c28114e2eb6 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -675,7 +675,7 @@ To prevent system resource from being exhausted by multiple concurrent streams, | Meaning | Whether to generate core file when server crashes | | Value Range | 0: false, 1: true | | Default Value | 1 | -| Note | The core file is generated under root directory `systemctl/launchctl start taosd` is used to start, or under the working directory if `taosd` is started directly on Linux/macOS Shell. | +| Note | The core file is generated under root directory `systemctl start taosd`/`launchctl start com.tdengine.taosd` is used to start, or under the working directory if `taosd` is started directly on Linux/macOS Shell. | ### udf diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index 6622ee525af1e696f21dcbef424269dd3c29e1a4..a938084eb1ee76e85c076b13335e0ce813bf7a7c 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -181,7 +181,7 @@ To make full use of the characteristics of time-series data, TDengine splits the For time-series data, there is generally a retention policy, which is determined by the system configuration parameter `keep`. Data files exceeding this set number of days will be automatically deleted by the system to free up storage space. -Given `duration` and `keep` parameters, the total number of data files in a vnode is: keep/duration. The total number of data files should not be too large or too small. 10 to 100 is appropriate. Based on this principle, reasonable `duration` can be set. In the current version, parameter `keep` can be modified, but parameter `duration` cannot be modified once it is set. +Given `duration` and `keep` parameters, the total number of data files in a vnode is: round up of (keep/duration+1). The total number of data files should not be too large or too small. 10 to 100 is appropriate. Based on this principle, reasonable `duration` can be set. In the current version, parameter `keep` can be modified, but parameter `duration` cannot be modified once it is set. In each data file, the data of a table is stored in blocks. A table can have one or more data file blocks. In a file block, data is stored in columns, occupying a continuous storage space, thus greatly improving the reading speed. The size of file block is determined by the system parameter `maxRows` (the maximum number of records per block), and the default value is 4096. This value should not be too large or too small. If it is too large, data location for queries will take a longer time. If it is too small, the index of data block is too large, and the compression efficiency will be low with slower reading speed. diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index cee036454ec4d3f4809576a1eee8ac054fcba056..836beb2417c310337d73e678f97810e447824b63 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -1,6 +1,58 @@ import taos -from taos.tmq import TaosConsumer -consumer = TaosConsumer('topic_ctb_column', group_id='vg2') -for msg in consumer: - for row in msg: - print(row) +from taos.tmq import * + +conn = taos.connect() + +print("init") +conn.execute("drop database if exists py_tmq") +conn.execute("create database if not exists py_tmq vgroups 2") +conn.select_db("py_tmq") +conn.execute( + "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)" +) +conn.execute("create table if not exists tb1 using stb1 tags(1)") +conn.execute("create table if not exists tb2 using stb1 tags(2)") +conn.execute("create table if not exists tb3 using stb1 tags(3)") + +print("create topic") +conn.execute("drop topic if exists topic_ctb_column") +conn.execute( + "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1" +) + +print("build consumer") +conf = TaosTmqConf() +conf.set("group.id", "tg2") +conf.set("td.connect.user", "root") +conf.set("td.connect.pass", "taosdata") +conf.set("enable.auto.commit", "true") + + +def tmq_commit_cb_print(tmq, resp, offset, param=None): + print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") + + +conf.set_auto_commit_cb(tmq_commit_cb_print, None) +tmq = conf.new_consumer() + +print("build topic list") + +topic_list = TaosTmqList() +topic_list.append("topic_ctb_column") + +print("basic consume loop") +tmq.subscribe(topic_list) + +sub_list = tmq.subscription() + +print("subscribed topics: ", sub_list) + +while 1: + res = tmq.poll(1000) + if res: + topic = res.get_topic_name() + vg = res.get_vgroup_id() + db = res.get_db_name() + print(f"topic: {topic}\nvgroup id: {vg}\ndb: {db}") + for row in res: + print(row) diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 50ee721386a591bbafece210cd963e50eef3ea7e..167a68d3c84a7da25f8ae0bfa9592fbd933299e0 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -188,13 +188,13 @@ Active: inactive (dead) -安装后，在应用程序目录下，双击 TDengine 图标来启动程序，也可以运行 `launchctl start taosd` 来启动 TDengine 服务进程。 +安装后，在应用程序目录下，双击 TDengine 图标来启动程序，也可以运行 `launchctl start com.tdengine.taosd` 来启动 TDengine 服务进程。 如下 `launchctl` 命令可以帮助你管理 TDengine 服务： -- 启动服务进程：`launchctl start taosd` +- 启动服务进程：`launchctl start com.tdengine.taosd` -- 停止服务进程：`launchctl stop taosd` +- 停止服务进程：`launchctl stop com.tdengine.taosd` - 查看服务状态：`launchctl list | grep taosd` diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 4c33b5232ac7d7511d1ff4bd121c389ce84d9a41..9346c3f76393cb4c6a36b129078c1c65e6869c28 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -340,7 +340,7 @@ LTRIM(expr) #### RTRIM ```sql -LTRIM(expr) +RTRIM(expr) ``` **功能说明**：返回清除右边空格后的字符串。 diff --git a/docs/zh/12-taos-sql/28-recovery.md b/docs/zh/12-taos-sql/28-recovery.md index 582c3739073513df4ceb212080805136947e62d4..b5088e7982620e43aaead01ad6493d1ef5ed72ab 100644 --- a/docs/zh/12-taos-sql/28-recovery.md +++ b/docs/zh/12-taos-sql/28-recovery.md @@ -17,10 +17,10 @@ conn_id 可以通过 `SHOW CONNECTIONS` 获取。 ## 终止查询 ```sql -SHOW QUERY query_id; +KILL QUERY kill_id; ``` -query_id 可以通过 `SHOW QUERIES` 获取。 +kill_id 可以通过 `SHOW QUERIES` 获取。 ## 终止事务 diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 7da8e9f331bd1d9c81e9bde441e35b03900422b0..1ab7f4f0169932576edf9087a825367573713b03 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -94,3 +94,10 @@ description: "TDengine 3.0 版本的语法变更说明" | 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 | 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时，超级表的数据会被合并成一条时间线。 + + +## SCHEMALESS 变更 + +| # | **元素** | **
差异性
** | **说明** | +| - | :------- | :-------- | :------- | +| 1 | 主键ts 变更为 _ts | 变更 | schemaless自动建的列名用 _ 开头，不同于2.x。 diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 9f4f728f78438bab299360717ba467a7fcea47ca..e067c646b23db8080251ab7b51e693337436cfcc 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -231,7 +231,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **name** : 数据库名。 -- **drop** : 插入前是否删除数据库，默认为 true。 +- **drop** : 插入前是否删除数据库，可选项为 "yes" 或者 "no", 为 "no" 时不创建。默认删除。 #### 流式计算相关配置参数 diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index 07d5b0692bc9ac53d8c46cc0542c9512c2aa536e..b89f625376404677af21f0d48b77c3df5ebd748a 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -115,7 +115,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 ## 数据模式映射规则 -本节将说明行协议的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为 +本节将说明 InfluxDB 行协议(Line Protocol)的数据如何映射成为具有模式的数据。每个行协议中数据 measurement 映射为 超级表名称。tag_set 中的 标签名称为 数据模式中的标签名，field_set 中的名称为列名称。以如下数据为例，说明映射规则： ```json diff --git a/examples/c/demoapi.c b/examples/c/demoapi.c index c3a6d267766dcb5621c126fb1dc3d411f9f3929a..18f1b5a0592a27b1f40c5c7f7c64e6e7341d1a6e 100644 --- a/examples/c/demoapi.c +++ b/examples/c/demoapi.c @@ -1,5 +1,17 @@ -// C api call sequence demo -// to compile: gcc -o apidemo apidemo.c -ltaos +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #include #include @@ -13,23 +25,23 @@ #include "taos.h" #define debugPrint(fmt, ...) \ - do { if (g_args.debug_print || g_args.verbose_print) \ - fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); } while(0) + do { if (g_args.debug_print || g_args.verbose_print) {\ + fprintf(stdout, "DEBG: "fmt, __VA_ARGS__); }} while (0) #define warnPrint(fmt, ...) \ do { fprintf(stderr, "\033[33m"); \ fprintf(stderr, "WARN: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) #define errorPrint(fmt, ...) \ do { fprintf(stderr, "\033[31m"); \ fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) #define okPrint(fmt, ...) \ do { fprintf(stderr, "\033[32m"); \ fprintf(stderr, "OK: "fmt, __VA_ARGS__); \ - fprintf(stderr, "\033[0m"); } while(0) + fprintf(stderr, "\033[0m"); } while (0) int64_t g_num_of_tb = 2; int64_t g_num_of_rec = 3; @@ -74,10 +86,16 @@ static void prepare_data(TAOS* taos) { res = taos_query(taos, "create database test;"); taos_free_result(res); taosMsleep(100); - taos_select_db(taos, "test"); + if (taos_select_db(taos, "test")) { + errorPrint("%s() LN%d: taos_select_db() failed\n", + __func__, __LINE__); + return; + } char command[1024] = {0}; - sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, city binary(20), dist nchar(20), street binary(20));"); + sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, " + "bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, " + "city binary(20), dist nchar(20), street binary(20));"); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { okPrint("%s created\n", "meters"); @@ -117,12 +135,11 @@ static void prepare_data(TAOS* taos) { sprintf(command, "insert into t%"PRId64" " "values(%" PRId64 ", %f, %"PRId64", " "'%c%d', '%s%c%d', '%c%d')", - i, 1650000000000+j, (float)j, j, + i, 1650000000000+j, j * 1.0, j, 'a'+(int)j%25, rand(), // "涛思", 'z' - (int)j%25, rand(), "TAOS", 'z' - (int)j%25, rand(), - 'b' - (int)j%25, rand() - ); + 'b' - (int)j%25, rand()); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { affected = taos_affected_rows(res); @@ -150,8 +167,7 @@ static void prepare_data(TAOS* taos) { i, 1650000000000+j+1, (float)j, j, 'a'+(int)j%25, rand(), "数据", 'z' - (int)j%25, rand(), - 'b' - (int)j%25, rand() - ); + 'b' - (int)j%25, rand()); res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { affected = taos_affected_rows(res); @@ -179,7 +195,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { } if (block) { - warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n"); + warnPrint("%s", "call taos_fetch_block(), " + "don't call taos_fetch_lengths()\n"); int rows = 0; while ((rows = taos_fetch_block(res, &row))) { int *lengths = taos_fetch_lengths(res); @@ -190,7 +207,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d type is %d, no need get offset\n", f, fields[f].type); for (int64_t c = 0; c < rows; c++) { - switch(fields[f].type) { + switch (fields[f].type) { case TSDB_DATA_TYPE_TIMESTAMP: if (taos_is_null(res, c, f)) { printf("col%d, row: %"PRId64" " @@ -199,7 +216,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %"PRId64"\n", f, c, - *(int64_t*)((char*)(row[f])+c*sizeof(int64_t))); + *(int64_t*)((char*)(row[f]) + +c*sizeof(int64_t))); } break; @@ -211,7 +229,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %d\n", f, c, - *(int32_t*)((char*)(row[f])+c*sizeof(int32_t))); + *(int32_t*)((char*)(row[f]) + +c*sizeof(int32_t))); } break; @@ -223,7 +242,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %f\n", f, c, - *(float*)((char*)(row[f])+c*sizeof(float))); + *(float*)((char*)(row[f]) + +c*sizeof(float))); } break; @@ -238,14 +258,18 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { if (offsets) { for (int c = 0; c < rows; c++) { if (offsets[c] != -1) { - int length = *(int16_t*)((char*)(row[f]) + offsets[c]); + int length = *(int16_t*)((char*)(row[f]) + + offsets[c]); char *buf = calloc(1, length + 1); - strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length); - printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n", + strncpy(buf, (char *)((char*)(row[f]) + + offsets[c] + 2), length); + printf("row: %d, col: %d, offset: %d, " + "length: %d, content: %s\n", c, f, offsets[c], length, buf); free(buf); } else { - printf("row: %d, col: %d, offset: -1, means content is NULL\n", + printf("row: %d, col: %d, offset: -1, " + "means content is NULL\n", c, f); } } @@ -267,7 +291,8 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { int* lengths = taos_fetch_lengths(res); if (lengths) { for (int c = 0; c < num_fields; c++) { - printf("row: %"PRId64", col: %d, is_null: %s, length of column %d is %d\n", + printf("row: %"PRId64", col: %d, is_null: %s, " + "length of column %d is %d\n", num_rows, c, taos_is_null(res, num_rows, c)?"True":"False", c, lengths[c]); @@ -277,7 +302,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { __func__, __LINE__, tbname); } - num_rows ++; + num_rows++; } } @@ -322,7 +347,8 @@ int main(int argc, char *argv[]) { #endif TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { - printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); + printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", + taos_errstr(taos)); exit(1); } diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 0e22985059907956c207d7c40103480d0913e65e..fd0521af3b96933dec7064483578e06d00731ce8 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -93,6 +93,7 @@ extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in // query client extern int32_t tsQueryPolicy; +extern int32_t tsQueryRspPolicy; extern int32_t tsQuerySmaOptimize; extern int32_t tsQueryRsmaTolerance; extern bool tsQueryPlannerTrace; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9f21ee007fde4d40527b05c4af129e2b2f54dfd3..748bd6afccb820f5db437acc74f6fbb88ad4ec6a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1902,7 +1902,7 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { if (pRebInfo == NULL) { return NULL; } - strcpy(pRebInfo->key, key); + tstrncpy(pRebInfo->key, key, sizeof(pRebInfo->key)); pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebInfo->lostConsumers == NULL) { goto _err; diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 0d43539629c170a2dcf22bee395cbc7caa93edc5..cdb1642a5c313767fe6a6d5ad53d6202f8c677a6 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -30,9 +30,10 @@ typedef struct SMnode SMnode; typedef struct { int32_t dnodeId; - bool standby; bool deploy; - SReplica replica; + int8_t selfIndex; + int8_t numOfReplicas; + SReplica replicas[TSDB_MAX_REPLICA]; SMsgCb msgCb; } SMnodeOpt; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c179d69960ac80eef69a04e721fb6c582560ef59..7d95e840e9c539aa7d75fa32a325de2be766aec2 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -54,6 +54,9 @@ typedef enum { #define QUERY_POLICY_QNODE 3 #define QUERY_POLICY_CLIENT 4 +#define QUERY_RSP_POLICY_DELAY 0 +#define QUERY_RSP_POLICY_QUICK 1 + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision @@ -230,7 +233,7 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, - void* (*mallocFp)(int32_t)); + void* (*mallocFp)(int64_t)); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); #define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 6322c6a1e969371c20efbb0e2f4b96800242ecee..ef8e80b57fbc7fdb6255bcd38b88652e20cc5a9d 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -43,7 +43,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type); int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); -int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow); +int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex, int32_t numOfRows); /* Math functions */ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 411a63a37966fc7104b86202fe97f22c78c92ad1..0f8220f19ccc1bc01a69c37a054f1621aef0e0c2 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -211,7 +211,7 @@ typedef struct SSyncInfo { int32_t syncInit(); void syncCleanUp(); -int64_t syncOpen(const SSyncInfo* pSyncInfo); +int64_t syncOpen(SSyncInfo* pSyncInfo); void syncStart(int64_t rid); void syncStop(int64_t rid); int32_t syncSetStandby(int64_t rid); @@ -233,7 +233,7 @@ const char* syncStr(ESyncState state); bool syncIsRestoreFinish(int64_t rid); int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot); -int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg); +int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg); // build SRpcMsg, need to call syncPropose with SRpcMsg int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 56c5750475e32b6c2606cfdbc0e4c5e75faabf46..53edcd4fee9bb9af6ab0d44235159b90990cf5c1 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -123,9 +123,9 @@ void rpcCleanup(); void *rpcOpen(const SRpcInit *pRpc); void rpcClose(void *); void rpcCloseImpl(void *); -void *rpcMallocCont(int32_t contLen); +void *rpcMallocCont(int64_t contLen); void rpcFreeCont(void *pCont); -void *rpcReallocCont(void *ptr, int32_t contLen); +void *rpcReallocCont(void *ptr, int64_t contLen); // Because taosd supports multi-process mode // These functions should not be used on the server side diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index de31a970dffc146dcc3b36e3933740200811af47..adf244e32a82a67386e4ef699b8689165c9519eb 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -43,6 +43,7 @@ extern "C" { #define WAL_FILE_LEN (WAL_PATH_LEN + 32) #define WAL_MAGIC 0xFAFBFCFDF4F3F2F1ULL #define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3) +#define WAL_RECOV_SIZE_LIMIT (100 * WAL_SCAN_BUF_SIZE) typedef enum { TAOS_WAL_WRITE = 1, diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 293d9d17f85b9fe890570ec7045a5b4c59b7adc5..d4e94d6173a813e44e68355bb6a0ead964823ba1 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -49,9 +49,15 @@ extern SDiskSpace tsTempSpace; void osDefaultInit(); void osUpdate(); void osCleanup(); + bool osLogSpaceAvailable(); bool osDataSpaceAvailable(); bool osTempSpaceAvailable(); + +bool osLogSpaceSufficient(); +bool osDataSpaceSufficient(); +bool osTempSpaceSufficient(); + void osSetTimezone(const char *timezone); void osSetSystemLocale(const char *inLocale, const char *inCharSet); @@ -59,4 +65,4 @@ void osSetSystemLocale(const char *inLocale, const char *inCharSet); } #endif -#endif /*_TD_OS_ENV_H_*/ \ No newline at end of file +#endif /*_TD_OS_ENV_H_*/ diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 3146590d6c9e81a415c7efce52ff16167f886b75..7ec8366ee4cb584dc41aa0ce94d5c54f6405964e 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -29,12 +29,12 @@ extern "C" { #define free FREE_FUNC_TAOS_FORBID #endif -void *taosMemoryMalloc(int32_t size); -void *taosMemoryCalloc(int32_t num, int32_t size); -void *taosMemoryRealloc(void *ptr, int32_t size); +void *taosMemoryMalloc(int64_t size); +void *taosMemoryCalloc(int64_t num, int64_t size); +void *taosMemoryRealloc(void *ptr, int64_t size); void *taosMemoryStrDup(const char *ptr); void taosMemoryFree(void *ptr); -int32_t taosMemorySize(void *ptr); +int64_t taosMemorySize(void *ptr); void taosPrintBackTrace(); #define taosMemoryFreeClear(ptr) \ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 6bc0e0e7dd509075834b5fb650b28206f5c502f9..bbddb539c60dd53d73ee6e7528caa5a4c48ce9c1 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -450,6 +450,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) +#define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006) // tfs #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) diff --git a/include/util/trbtree.h b/include/util/trbtree.h index f6d37e3d753de71fdf312b795935cb9014149f23..7c93c8dd8d65f2c4947019f370c0933e6ad81ee0 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -26,7 +26,7 @@ typedef struct SRBTree SRBTree; typedef struct SRBTreeNode SRBTreeNode; typedef struct SRBTreeIter SRBTreeIter; -typedef int32_t (*tRBTreeCmprFn)(const void *, const void *); +typedef int32_t (*tRBTreeCmprFn)(const SRBTreeNode *, const SRBTreeNode *); // SRBTree ============================================= #define tRBTreeMin(T) ((T)->min == ((T)->NIL) ? NULL : (T)->min) @@ -36,7 +36,7 @@ void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn); SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z); void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z); SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); -SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); +SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode); // SRBTreeIter ============================================= #define tRBTreeIterCreate(tree, ascend) \ @@ -53,8 +53,6 @@ struct SRBTreeNode { SRBTreeNode *right; }; -#define RBTREE_NODE_PAYLOAD(N) ((const void *)&(N)[1]) - struct SRBTree { tRBTreeCmprFn cmprFn; int64_t n; diff --git a/packaging/release.sh b/packaging/release.sh index 9ebbc50a84ffb762a2440b840fce8064bb4b7f2b..ce424918926dc7b9f62d70fe8da03d9688f33189 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -260,7 +260,7 @@ if [ "$osType" != "Darwin" ]; then if [[ "$pagMode" == "full" ]]; then if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then cd ${top_dir}/tools/taos-tools/packaging/deb - taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}') + taos_tools_ver=$(git tag |grep -v taos | sort | tail -1) [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" ${csudo}./make-taos-tools-deb.sh ${top_dir} \ @@ -285,7 +285,7 @@ if [ "$osType" != "Darwin" ]; then if [[ "$pagMode" == "full" ]]; then if [ -d ${top_dir}/tools/taos-tools/packaging/rpm ]; then cd ${top_dir}/tools/taos-tools/packaging/rpm - taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}' | sed -e 's/-/_/g') + taos_tools_ver=$(git tag |grep -v taos | sort | tail -1) [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" ${csudo}./make-taos-tools-rpm.sh ${top_dir} \ diff --git a/packaging/tools/TDengine b/packaging/tools/TDengine index 3025a5639f3dda2c9b06ef5ffaf0a7f0995c2532..c0b80564c49c5dde631bc96f68e4d37fc1f12b33 100755 --- a/packaging/tools/TDengine +++ b/packaging/tools/TDengine @@ -17,12 +17,12 @@ EOF taosd_status=`Launchctl list | grep taosd | head -n 1 | awk '{print $1}'` if [ "$taosd_status"x = "-"x ]; then - launchctl start taosd + launchctl start com.tdengine.taosd showAlertMessage "Taosd is running!" "TDengine" "ok" "note" else choose_result=`showAlertMessage "Taosd is running!\nDo you want to close it?" "TDengine" "yes,cancel" "stop"` if [ "$choose_result"x = "button returned:yes"x ]; then - launchctl stop taosd + launchctl stop com.tdengine.taosd fi fi diff --git a/packaging/tools/TDengine.pkgproj b/packaging/tools/TDengine.pkgproj new file mode 100644 index 0000000000000000000000000000000000000000..7aaac627352e60385af5fd6e4fd2b2694952a580 --- /dev/null +++ b/packaging/tools/TDengine.pkgproj @@ -0,0 +1,882 @@ + + + + + PACKAGES + + + MUST-CLOSE-APPLICATION-ITEMS + + MUST-CLOSE-APPLICATIONS + + PACKAGE_FILES + + DEFAULT_INSTALL_LOCATION + / + HIERARCHY + + CHILDREN + + + CHILDREN + + GID + 80 + PATH + Applications + PATH_TYPE + 0 + PERMISSIONS + 509 + TYPE + 1 + UID + 0 + + + CHILDREN + + + CHILDREN + + GID + 80 + PATH + Application Support + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Automator + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Documentation + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Extensions + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Filesystems + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Frameworks + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Input Methods + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Internet Plug-Ins + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Keyboard Layouts + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + LaunchAgents + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + LaunchDaemons + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + PreferencePanes + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Preferences + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 80 + PATH + Printers + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + PrivilegedHelperTools + PATH_TYPE + 0 + PERMISSIONS + 1005 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + QuickLook + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + QuickTime + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Screen Savers + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Scripts + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Services + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Widgets + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + GID + 0 + PATH + Library + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + + CHILDREN + + GID + 0 + PATH + Shared + PATH_TYPE + 0 + PERMISSIONS + 1023 + TYPE + 1 + UID + 0 + + + GID + 80 + PATH + Users + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + GID + 0 + PATH + / + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + PAYLOAD_TYPE + 0 + PRESERVE_EXTENDED_ATTRIBUTES + + SHOW_INVISIBLE + + SPLIT_FORKS + + TREAT_MISSING_FILES_AS_WARNING + + VERSION + 5 + + PACKAGE_SCRIPTS + + POSTINSTALL_PATH + + PATH + /opt/taos/TDengine/packaging/tools/post.sh + PATH_TYPE + 0 + + PREINSTALL_PATH + + PATH_TYPE + 0 + + RESOURCES + + + CHILDREN + + GID + 0 + PATH + /opt/tdengine + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 3 + UID + 0 + + + + PACKAGE_SETTINGS + + AUTHENTICATION + 1 + CONCLUSION_ACTION + 0 + FOLLOW_SYMBOLIC_LINKS + + IDENTIFIER + com.taosdata.pkg.pkgtaos + LOCATION + 0 + NAME + TDengine + OVERWRITE_PERMISSIONS + + PAYLOAD_SIZE + -1 + REFERENCE_PATH + + RELOCATABLE + + USE_HFS+_COMPRESSION + + VERSION + 3.0.1.4 + + TYPE + 0 + UUID + 48F6B249-AF56-46E3-B75A-AEED5858A764 + + + PROJECT + + PROJECT_COMMENTS + + NOTES + + + + PROJECT_PRESENTATION + + BACKGROUND + + APPAREANCES + + DARK_AQUA + + LIGHT_AQUA + + + SHARED_SETTINGS_FOR_ALL_APPAREANCES + + + INSTALLATION TYPE + + HIERARCHIES + + INSTALLER + + LIST + + + CHILDREN + + DESCRIPTION + + OPTIONS + + HIDDEN + + STATE + 1 + + PACKAGE_UUID + 48F6B249-AF56-46E3-B75A-AEED5858A764 + TITLE + + TYPE + 0 + UUID + 7ED88D2C-D55C-46FF-99CB-73313ACAD73D + + + REMOVED + + + + MODE + 0 + + INSTALLATION_STEPS + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewIntroductionController + INSTALLER_PLUGIN + Introduction + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewReadMeController + INSTALLER_PLUGIN + ReadMe + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewLicenseController + INSTALLER_PLUGIN + License + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewDestinationSelectController + INSTALLER_PLUGIN + TargetSelect + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewInstallationTypeController + INSTALLER_PLUGIN + PackageSelection + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewInstallationController + INSTALLER_PLUGIN + Install + LIST_TITLE_KEY + InstallerSectionTitle + + + ICPRESENTATION_CHAPTER_VIEW_CONTROLLER_CLASS + ICPresentationViewSummaryController + INSTALLER_PLUGIN + Summary + LIST_TITLE_KEY + InstallerSectionTitle + + + INTRODUCTION + + LOCALIZATIONS + + + LANGUAGE + English + VALUE + + PATH + /opt/taos/TDengine/packaging/tools/mac_before_install.txt + PATH_TYPE + 0 + + + + + LICENSE + + LOCALIZATIONS + + MODE + 0 + + README + + LOCALIZATIONS + + + SUMMARY + + LOCALIZATIONS + + + TITLE + + LOCALIZATIONS + + + LANGUAGE + English + VALUE + TDengine + + + + + PROJECT_REQUIREMENTS + + LIST + + RESOURCES + + ROOT_VOLUME_ONLY + + + PROJECT_SETTINGS + + BUILD_FORMAT + 0 + BUILD_PATH + + PATH + /opt/taos/TDengine/release + PATH_TYPE + 0 + + EXCLUDED_FILES + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + .DS_Store + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove .DS_Store files + PROXY_TOOLTIP + Remove ".DS_Store" files created by the Finder. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + .pbdevelopment + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove .pbdevelopment files + PROXY_TOOLTIP + Remove ".pbdevelopment" files created by ProjectBuilder or Xcode. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + CVS + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .cvsignore + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + .cvspass + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + .svn + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .git + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .gitignore + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove SCM metadata + PROXY_TOOLTIP + Remove helper files and folders used by the CVS, SVN or Git Source Code Management systems. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + classes.nib + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + designable.db + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + info.nib + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Optimize nib files + PROXY_TOOLTIP + Remove "classes.nib", "info.nib" and "designable.nib" files within .nib bundles. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + Resources Disabled + TYPE + 1 + + + PROTECTED + + PROXY_NAME + Remove Resources Disabled folders + PROXY_TOOLTIP + Remove "Resources Disabled" folders. + STATE + + + + SEPARATOR + + + + NAME + TDengine-client-3.0.1.4-macOS-arm64 + PAYLOAD_ONLY + + TREAT_MISSING_PRESENTATION_DOCUMENTS_AS_WARNING + + + + TYPE + 0 + VERSION + 2 + + diff --git a/packaging/tools/com.taosdata.taosd.plist b/packaging/tools/com.taosdata.taosd.plist index ca9b8c5ef1a103afc125417f99b60ccf4b51f610..0bf3135ac30239fefe081d1b45928540ecbebdbf 100644 --- a/packaging/tools/com.taosdata.taosd.plist +++ b/packaging/tools/com.taosdata.taosd.plist @@ -3,7 +3,7 @@ Label - taosd + com.tdengine.taosd ProgramArguments /usr/local/bin/taosd diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt index f70ea7161f4df28594639f8f2a1f0a335b8ad03f..b33cd478b8982a5c6c8263ea3d2ea6d607f3f706 100644 --- a/packaging/tools/mac_before_install.txt +++ b/packaging/tools/mac_before_install.txt @@ -1,5 +1,5 @@ TDengine is a high-efficient, scalable, high-available distributed time-series database, which makes a lot of optimizations on inserting and querying data, which is far more efficient than normal regular databases. So TDengine can meet the high requirements of IOT and other areas on storing and querying a large amount of data. To configure TDengine : edit /etc/taos/taos.cfg -To start service : launchctl start taosd +To start service : launchctl start com.tdengine.taosd To access TDengine : use taos in shell \ No newline at end of file diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index b8796ddf09b2f0d899aad793ebe757b649ad9814..b56a5a203d7e42827512c8930345e45022a719b1 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -615,7 +615,7 @@ function update_TDengine() { if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start ${serverName}${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" fi [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" @@ -666,7 +666,7 @@ function install_TDengine() { if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" else - echo -e "${GREEN_DARK}To start service ${NC}: launchctl start ${serverName}${NC}" + echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" fi [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 29d4ac017ca9d8813e1f7c69f4f5e6056602b16c..b729513b97206673c2ed6d2d9be20a28b5b8e258 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -18,6 +18,7 @@ if [ "$osType" != "Darwin" ]; then script_dir=$(dirname $(readlink -f "$0")) verNumber="" lib_file_ext="so" + lib_file_ext_1="so.1" bin_link_dir="/usr/bin" lib_link_dir="/usr/lib" @@ -29,6 +30,7 @@ else script_dir=${source_dir}/packaging/tools verNumber=`ls tdengine/driver | grep -E "libtaos\.[0-9]\.[0-9]" | sed "s/libtaos.//g" | sed "s/.dylib//g" | head -n 1` lib_file_ext="dylib" + lib_file_ext_1="1.dylib" bin_link_dir="/usr/local/bin" lib_link_dir="/usr/local/lib" @@ -134,14 +136,14 @@ function install_lib() { [ -f ${lib_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.${lib_file_ext} || : [ -f ${lib64_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.${lib_file_ext} || : - ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 - ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.${lib_file_ext_1} + ${csudo}ln -s ${lib_link_dir}/libtaos.${lib_file_ext_1} ${lib_link_dir}/libtaos.${lib_file_ext} [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib_link_dir}/libtaosws.${lib_file_ext} ||: - if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then - ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.${lib_file_ext} ]]; then + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.${lib_file_ext_1} || : + ${csudo}ln -s ${lib64_link_dir}/libtaos.${lib_file_ext_1} ${lib64_link_dir}/libtaos.${lib_file_ext} || : [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} || : fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 6172fc9bc8bcce6a311b82cff52b797c3731b296..bace0bc869e599aaf376356094b85e8f5e0d4556 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -117,7 +117,7 @@ function clean_local_bin() { function clean_lib() { # Remove link ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : + [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index e8e3c878496c58631131922cc6de47491d548f06..0c445c7fbfa01b6432259bd20e7e481966aee7a7 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -19,6 +19,13 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) +if(TD_DARWIN_ARM64) + target_link_libraries( + taos + PRIVATE "-arch x86_64" + ) +endif() + if(TD_WINDOWS) INCLUDE_DIRECTORIES(jni/windows) INCLUDE_DIRECTORIES(jni/windows/win32) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 096e2cdac8b892673de7e991f64a60dcb7dc8f6d..3f24a08cf4a68a19407da670fc43fe2be3bea279 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -173,7 +173,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; - tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, + tscTrace("conn %u hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes); if (pRsp->query->killRid) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7b437e47699ca623da62ceb496ab2a7337df7412..b494140da59e9ac410c1c6bb649894b30edb6d2d 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -186,7 +186,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, STscObj* pTscObj = (*pRequest)->pTscObj; if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self))) { - tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, + tscError("%" PRIx64 " failed to add to request container, reqId:0x%" PRIu64 ", conn:%" PRIx64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); taosMemoryFree(param); @@ -371,7 +371,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) { pInfo->pQnodeList = taosArrayDup(pNodeList); taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad); tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId, - taosArrayGetSize(pInfo->pQnodeList)); + (int)taosArrayGetSize(pInfo->pQnodeList)); } taosThreadMutexUnlock(&pInfo->qnodeMutex); @@ -1036,30 +1036,38 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat pRequest->type = pQuery->msgType; SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); - SPlanContext cxt = {.queryId = pRequest->requestId, - .acctId = pRequest->pTscObj->acctId, - .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), - .pAstRoot = pQuery->pRoot, - .showRewrite = pQuery->showRewrite, - .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, - .pUser = pRequest->pTscObj->user, - .sysInfo = pRequest->pTscObj->sysInfo, - .allocatorId = pRequest->allocatorRefId}; - SQueryPlan* pDag = NULL; - int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); - if (code) { - tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), - pRequest->requestId); - } else { - pRequest->body.subplanNum = pDag->numOfSubplans; - } - pRequest->metric.planEnd = taosGetTimestampUs(); + SPlanContext cxt = {.queryId = pRequest->requestId, + .acctId = pRequest->pTscObj->acctId, + .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite, + .pMsg = pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pUser = pRequest->pTscObj->user, + .sysInfo = pRequest->pTscObj->sysInfo, + .allocatorId = pRequest->allocatorRefId}; + + SAppInstInfo* pAppInfo = getAppInfo(pRequest); + SQueryPlan* pDag = NULL; + + int64_t st = taosGetTimestampUs(); + int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); + if (code) { + tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + } else { + pRequest->body.subplanNum = pDag->numOfSubplans; + } - if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { - SArray* pNodeList = NULL; - buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); + pRequest->metric.planEnd = taosGetTimestampUs(); + if (code == TSDB_CODE_SUCCESS) { + tscDebug("0x%" PRIx64 " create query plan success, elapsed time:%.2f ms, 0x%" PRIx64, pRequest->self, + (pRequest->metric.planEnd - st)/1000.0, pRequest->requestId); + } + if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { + SArray* pNodeList = NULL; + buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter, .requestId = pRequest->requestId, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 6f8cef7c0d50aff6dd76903691a0fc3521230d85..6126817ececa647dacd220031163464ae57d854e 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -146,7 +146,7 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return terrno; } @@ -154,13 +154,12 @@ int taos_errno(TAOS_RES *res) { return 0; } - return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL - : ((SRequestObj *)res)->code; + return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return (const char *)tstrerror(terrno); } @@ -172,7 +171,7 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { - return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL) + return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY) : (const char *)tstrerror(pRequest->code); } } @@ -222,7 +221,9 @@ void taos_kill_query(TAOS *taos) { int64_t rid = *(int64_t *)taos; STscObj *pTscObj = acquireTscObj(rid); - stopAllRequests(pTscObj->pRequests); + if (pTscObj) { + stopAllRequests(pTscObj->pRequests); + } releaseTscObj(rid); } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index a25733593143787e937f719302754babee58463e..1dc319be94e8ab880aaf87259f8b7166a900e871 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -410,6 +410,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVAlterTbReq vAlterTbReq = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -419,7 +420,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -524,6 +525,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropStbReq req = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -533,7 +535,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -556,6 +558,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropTbBatchReq req = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -565,7 +568,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -684,7 +687,7 @@ end: static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { SVDropStbReq req = {0}; - SDecoder coder; + SDecoder coder = {0}; SMDropStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; @@ -1212,6 +1215,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; + SSubmitReq* subReq = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); if (!pRequest) { @@ -1228,8 +1232,8 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - strcpy(pName.dbname, pRequest->pDb); - strcpy(pName.tname, tbname); + tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); + tstrncpy(pName.tname, tbname, sizeof(pName.tname)); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); @@ -1278,7 +1282,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; int32_t totalLen = sizeof(SSubmitReq) + submitLen; - SSubmitReq* subReq = taosMemoryCalloc(1, totalLen); + subReq = taosMemoryCalloc(1, totalLen); SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); @@ -1352,6 +1356,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) if (NULL == pQuery) { uError("create SQuery error"); code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(subReq); goto end; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; @@ -1390,6 +1395,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); + taosMemoryFree(subReq); return code; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c5633e73d0521c9304f1f23c778506cd9a0ad005..5c37822222ece2e83db0d958074d7e36913a35ea 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -299,6 +299,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { + taosHashCleanup(hashTmp); return -1; } } @@ -430,7 +431,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { STableMeta *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); SRequestConnInfo conn = {0}; conn.pTrans = info->taos->pAppInfo->pTransporter; @@ -874,7 +875,8 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra kv->i = ts; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); + return TSDB_CODE_SUCCESS; } @@ -1009,6 +1011,7 @@ static void smlParseTelnetElement(const char **sql, const char **data, int32_t * static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { + if(!cols) return TSDB_CODE_OUT_OF_MEMORY; const char *sql = data; size_t childTableNameLen = strlen(tsSmlChildTableName); while (*sql != '\0') { @@ -1082,7 +1085,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab kv->length = valueLen; kv->type = TSDB_DATA_TYPE_NCHAR; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); } return TSDB_CODE_SUCCESS; @@ -1370,8 +1373,14 @@ static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { SHashObj *s2 = *(SHashObj **)key2; SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); + if(!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP){ + uError("smlKvTimeHashCompare kv1"); + return -1; + } + if(!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP){ + uError("smlKvTimeHashCompare kv2"); + return -1; + } if (kv1->i < kv2->i) { return -1; } else if (kv1->i > kv2->i) { @@ -1735,7 +1744,7 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { kv->i = tsVal; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); return TSDB_CODE_SUCCESS; } @@ -1932,6 +1941,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { } static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { + if(!cols) return TSDB_CODE_OUT_OF_MEMORY; cJSON *metricVal = cJSON_GetObjectItem(root, "value"); if (metricVal == NULL) { return TSDB_CODE_TSC_INVALID_JSON; @@ -1941,7 +1951,7 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { if (!kv) { return TSDB_CODE_OUT_OF_MEMORY; } - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); kv->key = VALUE; kv->keyLen = VALUE_LEN; @@ -1955,7 +1965,9 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { int32_t ret = TSDB_CODE_SUCCESS; - + if (!pKVs){ + return TSDB_CODE_OUT_OF_MEMORY; + } cJSON *tags = cJSON_GetObjectItem(root, "tags"); if (tags == NULL || tags->type != cJSON_Object) { return TSDB_CODE_TSC_INVALID_JSON; @@ -1985,14 +1997,14 @@ static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableN return TSDB_CODE_TSC_INVALID_JSON; } memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); + tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); continue; } // add kv to SSmlKv SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (pKVs) taosArrayPush(pKVs, &kv); + taosArrayPush(pKVs, &kv); // key kv->keyLen = keyLen; @@ -2103,6 +2115,8 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { if (!oneTable) { tinfo = smlBuildTableInfo(); if (!tinfo) { + smlDestroyCols(cols); + if (info->dataFormat) taosArrayDestroy(cols); return TSDB_CODE_TSC_OUT_OF_MEMORY; } taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); @@ -2295,7 +2309,7 @@ static int32_t smlInsertData(SSmlHandle *info) { SSmlTableInfo *tableData = *oneTable; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); SRequestConnInfo conn = {0}; @@ -2477,11 +2491,12 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { } else { pParam->request->body.resInfo.numOfRows += info->affectedRows; } + // unlock + taosThreadSpinUnlock(&pParam->lock); + if (pParam->cnt == pParam->total) { tsem_post(&pParam->sem); } - taosThreadSpinUnlock(&pParam->lock); - // unlock uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows); info->cost.endTime = taosGetTimestampUs(); info->cost.code = code; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 7a800bd3349308d87ef4897e034b23330dabce0e..bf3fd00f1445aef67a7f8336271976dc539035b6 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -201,6 +201,9 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if(!pSrc){ + return TSDB_CODE_OUT_OF_MEMORY; + } STableDataBlocks* pDst = NULL; STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 66f992f05f94f1e583118044cea2271ec8fcd7ee..02aa7b806c9a9502c9b838c897f6d4cde5216527 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -709,6 +709,7 @@ void tmqSendHbReq(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { + taosMemoryFree(param); return; } int64_t consumerId = tmq->consumerId; @@ -870,8 +871,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("consumer %" PRId64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), - pTmq->groupId); + tscError("consumer setup failed since %s", terrstr()); return NULL; } @@ -939,10 +939,9 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { return NULL; } - int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); - *pRefId = pTmq->refId; - if (pTmq->hbBgEnable) { + int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); + *pRefId = pTmq->refId; pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer); } @@ -966,6 +965,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = -1; + tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz); + req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); @@ -1297,7 +1298,8 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { pParam->code = code; if (code != 0) { - tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async); + tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d, code %x", tmq->consumerId, + pParam->async, code); goto END; } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index e69174476a8631d5a1935764a03bf1645dd1102b..21a52a4b57f9e7ba9cda24e8f9a08fdb53545b1c 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -52,15 +52,15 @@ void printResult(TAOS_RES* pRes) { int32_t n = 0; char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t* length = taos_fetch_lengths(pRes); - for (int32_t i = 0; i < numOfFields; ++i) { - printf("(%d):%d ", i, length[i]); - } - printf("\n"); - - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - memset(str, 0, sizeof(str)); +// int32_t* length = taos_fetch_lengths(pRes); +// for(int32_t i = 0; i < numOfFields; ++i) { +// printf("(%d):%d " , i, length[i]); +// } +// printf("\n"); +// +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// memset(str, 0, sizeof(str)); } } @@ -102,17 +102,6 @@ void queryCallback(void* param, void* res, int32_t code) { taos_fetch_raw_block_a(res, fetchCallback, param); } -void queryCallback1(void* param, void* res, int32_t code) { - if (code != TSDB_CODE_SUCCESS) { - printf("failed to execute, reason:%s\n", taos_errstr(res)); - } - - taos_free_result(res); - - printf("exec query:\n"); - taos_query_a(param, "select * from tm1", queryCallback, param); -} - void createNewTable(TAOS* pConn, int32_t index) { char str[1024] = {0}; sprintf(str, "create table tu%d using st2 tags(%d)", index, index); @@ -123,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for (int32_t i = 0; i < 3280; i += 20) { + for(int32_t i = 0; i < 10000; i += 20) { char sql[1024] = {0}; sprintf(sql, "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" @@ -141,10 +130,49 @@ void createNewTable(TAOS* pConn, int32_t index) { taos_free_result(p); } } + +void *queryThread(void *arg) { + TAOS* pConn = taos_connect("192.168.0.209", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + printf("failed to connect to db, reason:%s", taos_errstr(pConn)); + return NULL; + } + + int64_t el = 0; + + for (int32_t i = 0; i < 5000000; ++i) { + int64_t st = taosGetTimestampUs(); + TAOS_RES* pRes = taos_query(pConn, + "SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;"); + if (taos_errno(pRes) != 0) { + printf("failed, reason:%s\n", taos_errstr(pRes)); + } else { + printResult(pRes); + } + + taos_free_result(pRes); + el += (taosGetTimestampUs() - st); + if (i % 1000 == 0 && i != 0) { + printf("total:%d, avg time:%.2fms\n", i, el/(double)(i*1000)); + } + } + + taos_close(pConn); + return NULL; +} + +static int32_t numOfThreads = 1; } // namespace int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); + if (argc > 1) { + numOfThreads = atoi(argv[1]); + } + + numOfThreads = TMAX(numOfThreads, 1); + printf("the runing threads is:%d", numOfThreads); + return RUN_ALL_TESTS(); } @@ -664,7 +692,6 @@ TEST(testCase, insert_test) { taos_free_result(pRes); taos_close(pConn); } -#endif TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -676,13 +703,14 @@ TEST(testCase, projection_query_tables) { // } // taos_free_result(pRes); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); + TAOS_RES* pRes = taos_query(pConn, "use benchmarkcpu"); taos_free_result(pRes); pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } + taos_free_result(pRes); pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); @@ -722,6 +750,16 @@ TEST(testCase, projection_query_tables) { taos_free_result(pRes); taos_close(pConn); } +#endif + +TEST(testCase, tsbs_perf_test) { + TdThread qid[20] = {0}; + + for(int32_t i = 0; i < numOfThreads; ++i) { + taosThreadCreate(&qid[i], NULL, queryThread, NULL); + } + getchar(); +} #if 0 TEST(testCase, projection_query_stables) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ad5772e0fe534c7c5ac3e795abf476084f734ebb..269e72f15184e6309f3a9256ce465ee7e7f42dc0 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -916,6 +916,10 @@ char *tTagValToData(const STagVal *value, bool isJson) { } bool tTagGet(const STag *pTag, STagVal *pTagVal) { + if(!pTag || !pTagVal){ + return false; + } + int16_t lidx = 0; int16_t ridx = pTag->nTag - 1; int16_t midx; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 88df54a9eace1c310af0c0e9787d8a8774803e97..56daa4075a4ca244da17bbd6d32a51b906856393 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -90,6 +90,7 @@ bool tsSmlDataFormat = false; // query int32_t tsQueryPolicy = 1; +int32_t tsQueryRspPolicy = 0; int32_t tsQuerySmaOptimize = 0; int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; @@ -350,6 +351,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0) != 0) return -1; if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1; if (cfgAddInt32(pCfg, "mnodeShmSize", tsMnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; @@ -728,6 +730,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; + tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index a4c37dd334984ff5e834463ec802b937ff557460..17cbde437b2e94d8d111c76600ddef8fb7767818 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -41,8 +41,8 @@ typedef struct SMnodeMgmt { } SMnodeMgmt; // mmFile.c -int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed); -int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed); +int32_t mmReadFile(const char *path, SMnodeOpt *pOption); +int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption); // mmHandle.c SArray *mmGetMsgHandles(); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 27a35ae17a859ba1f5fd7289e04c921c775b904a..c5ddb9f021c65a2ac7f616a9236bced7954a2aa0 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mmInt.h" -int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { +int32_t mmReadFile(const char *path, SMnodeOpt *pOption) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int32_t len = 0; int32_t maxLen = 4096; @@ -25,7 +25,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { char file[PATH_MAX] = {0}; TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP); pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { code = 0; @@ -50,38 +50,69 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { dError("failed to read %s since deployed not found", file); goto _OVER; } - *pDeployed = deployed->valueint; + pOption->deploy = deployed->valueint; - cJSON *id = cJSON_GetObjectItem(root, "id"); - if (id) { - if (id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); + cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex"); + if (selfIndex) { + if (selfIndex->type != cJSON_Number) { + dError("failed to read %s since selfIndex not found", file); goto _OVER; } - if (pReplica) { - pReplica->id = id->valueint; - } + pOption->selfIndex = selfIndex->valueint; } - cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn"); - if (fqdn) { - if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); + cJSON *replicas = cJSON_GetObjectItem(root, "replicas"); + if (replicas) { + if (replicas->type != cJSON_Array) { + dError("failed to read %s since replicas not found", file); goto _OVER; } - if (pReplica) { - tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); - } - } - cJSON *port = cJSON_GetObjectItem(root, "port"); - if (port) { - if (port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); + int32_t numOfReplicas = cJSON_GetArraySize(replicas); + if (numOfReplicas <= 0) { + dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas); goto _OVER; } - if (pReplica) { - pReplica->port = (uint16_t)port->valueint; + pOption->numOfReplicas = numOfReplicas; + + for (int32_t i = 0; i < numOfReplicas; ++i) { + SReplica *pReplica = pOption->replicas + i; + + cJSON *replica = cJSON_GetArrayItem(replicas, i); + if (replica == NULL) break; + + cJSON *id = cJSON_GetObjectItem(replica, "id"); + if (id) { + if (id->type != cJSON_Number) { + dError("failed to read %s since id not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->id = id->valueint; + } + } + + cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn"); + if (fqdn) { + if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { + dError("failed to read %s since fqdn not found", file); + goto _OVER; + } + if (pReplica) { + tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); + } + } + + cJSON *port = cJSON_GetObjectItem(replica, "port"); + if (port) { + if (port->type != cJSON_Number) { + dError("failed to read %s since port not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->port = (uint16_t)port->valueint; + } + } } } @@ -92,18 +123,18 @@ _OVER: if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); if (code == 0) { - dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy); } terrno = code; return code; } -int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) { +int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; - snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); - snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP); TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { @@ -117,12 +148,25 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - if (pReplica != NULL && pReplica->id > 0) { - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n,", pReplica->port); + if (pOption->deploy && pOption->numOfReplicas > 0) { + len += snprintf(content + len, maxLen - len, " \"selfIndex\": %d,\n", pOption->selfIndex); + len += snprintf(content + len, maxLen - len, " \"replicas\": [{\n"); + + for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + const SReplica *pReplica = pOption->replicas + i; + if (pReplica != NULL && pReplica->id > 0) { + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); + } + if (i < pOption->numOfReplicas - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }],\n"); + } + } } - len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); + len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", pOption->deploy); len += snprintf(content + len, maxLen - len, "}\n"); taosWriteFile(pFile, content, len); @@ -136,6 +180,6 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) return -1; } - dDebug("successed to write %s, deployed:%d", realfile, deployed); + dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy); return 0; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index ec761e6441ce651db5d6f3b034e4f11b1686fcdc..fc45dbf15f0630cf505db8cf013ff6297f27eb9e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -80,18 +80,21 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - if (createReq.replica != 1) { + SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1}; + memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas)); + for (int32_t i = 0; i < option.numOfReplicas; ++i) { + if (createReq.replicas[i].id == pInput->pData->dnodeId) { + option.selfIndex = i; + } + } + + if (option.selfIndex == -1) { terrno = TSDB_CODE_INVALID_OPTION; - dGError("failed to create mnode since %s", terrstr()); + dGError("failed to create mnode since %s, selfIndex is -1", terrstr()); return -1; } - bool deployed = true; - - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) { + if (mmWriteFile(pInput->path, &option) != 0) { dGError("failed to write mnode file since %s", terrstr()); return -1; } @@ -113,12 +116,8 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - bool deployed = false; - - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, NULL, deployed) != 0) { + SMnodeOpt option = {.deploy = false}; + if (mmWriteFile(pInput->path, &option) != 0) { dGError("failed to write mnode file since %s", terrstr()); return -1; } @@ -207,7 +206,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - // if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 49207225a5d4877eb138a75fb89bb2c5e7f1e474..e7f71ad4200bd2b1c3c5fcf10e82aa9aa0ea417f 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -25,38 +25,35 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) { } static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { - SMnodeMgmt mgmt = {0}; - mgmt.path = pInput->path; - if (mmReadFile(&mgmt, NULL, required) != 0) { + SMnodeOpt option = {0}; + if (mmReadFile(pInput->path, &option) != 0) { return -1; } - if (!(*required)) { + if (!option.deploy) { *required = mmDeployRequired(pInput); + } else { + *required = true; } return 0; } static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { - pOption->standby = false; pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->replica.id = 1; - pOption->replica.port = tsServerPort; - tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->selfIndex = 0; + pOption->numOfReplicas = 1; + pOption->replicas[0].id = 1; + pOption->replicas[0].port = tsServerPort; + tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); } -static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) { - pOption->standby = false; +static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - if (pReplica->id > 0) { - pOption->standby = true; - pOption->replica = *pReplica; - } } static void mmClose(SMnodeMgmt *pMgmt) { @@ -95,22 +92,20 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); - bool deployed = false; - SReplica replica = {0}; - if (mmReadFile(pMgmt, &replica, &deployed) != 0) { + SMnodeOpt option = {0}; + if (mmReadFile(pMgmt->path, &option) != 0) { dError("failed to read file since %s", terrstr()); mmClose(pMgmt); return -1; } - SMnodeOpt option = {0}; - if (!deployed) { + if (!option.deploy) { dInfo("mnode start to deploy"); pMgmt->pData->dnodeId = 1; mmBuildOptionForDeploy(pMgmt, pInput, &option); } else { dInfo("mnode start to open"); - mmBuildOptionForOpen(pMgmt, &replica, &option); + mmBuildOptionForOpen(pMgmt, &option); } pMgmt->pMnode = mndOpen(pMgmt->path, &option); @@ -128,9 +123,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("mnode-worker", "initialized"); - if (!deployed || replica.id > 0) { - deployed = true; - if (mmWriteFile(pMgmt, NULL, deployed) != 0) { + if (option.numOfReplicas > 0) { + option.deploy = true; + option.numOfReplicas = 0; + if (mmWriteFile(pMgmt->path, &option) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index d2b071ec61d7343f826ed9975cd6b07745399357..849d3ef39019fcb5ce9dfb987cac4772e91839a7 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -72,7 +72,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { mndPostProcessQueryMsg(pMsg); } - dGTrace("msg:%p, is freed, code:0x%x", pMsg, code); + dGTrace("msg:%p is freed, code:%s", pMsg, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index d07ec7abb03501f1fe627f66dce36228df014fad..076826ebc2d0956aad14f8eeec27a574425788b1 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -51,26 +51,14 @@ static int32_t dmInitMonitor() { static bool dmCheckDiskSpace() { osUpdate(); - if (!osDataSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least , quit", - (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osDataSpaceSufficient()) { + dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); } - if (!osLogSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least, quit", - (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osLogSpaceSufficient()) { + dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least", (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); } - if (!osTempSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least, quit", - (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osTempSpaceSufficient()) { + dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); } return true; } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 71315a39fd6c75d7b09e7371a9e0af30577a54d8..f55e830a44c8d341e25cd294ee31c56ed7707277 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -87,12 +87,13 @@ typedef struct { typedef struct { tsem_t syncSem; int64_t sync; - SReplica replica; int32_t errCode; int32_t transId; SRWLatch lock; - int8_t standby; int8_t leaderTransferFinish; + int8_t selfIndex; + int8_t numOfReplicas; + SReplica replicas[TSDB_MAX_REPLICA]; } SSyncMgmt; typedef struct { @@ -130,11 +131,10 @@ typedef struct SMnode { void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); int64_t mndGenerateUid(const char *name, int32_t len); -int32_t mndAcquireRpcRef(SMnode *pMnode); -void mndReleaseRpcRef(SMnode *pMnode); -void mndSetRestore(SMnode *pMnode, bool restored); -void mndSetStop(SMnode *pMnode); -bool mndGetStop(SMnode *pMnode); +void mndSetRestored(SMnode *pMnode, bool restored); +bool mndGetRestored(SMnode *pMnode); +void mndSetStop(SMnode *pMnode); +bool mndGetStop(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSync.h b/source/dnode/mnode/impl/inc/mndSync.h index cb9d70d5ee48f542dbe58100328b7f2284ea2926..993efbcd084443fccd8ef9776b942e7af671bf60 100644 --- a/source/dnode/mnode/impl/inc/mndSync.h +++ b/source/dnode/mnode/impl/inc/mndSync.h @@ -24,7 +24,7 @@ extern "C" { int32_t mndInitSync(SMnode *pMnode); void mndCleanupSync(SMnode *pMnode); -bool mndIsMaster(SMnode *pMnode); +bool mndIsLeader(SMnode *pMnode); int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId); void mndSyncStart(SMnode *pMnode); void mndSyncStop(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 6670211d7834456af6ed4456881dea149e4e0414..3b3187385771da81d703a270c3f9f07e3171505e 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1809,7 +1809,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc } while (numOfRows < rowsCapacity) { - pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus); + pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus, true); if (pShow->pIter == NULL) break; if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 9ade39e8e7c3302389e8f4b339baaad6071ae5b3..fba1fd94d6b537a9f587d898c062f01a0b47df19 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -423,7 +423,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { goto _OVER; } else { pDnode->accessTimes++; - mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); + mDebug("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); } } @@ -471,6 +471,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } pDnode->lastAccessTime = curMs; + pDnode->accessTimes++; code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 275fc76e360c19ffb56e76d639a1319ac17d0c4f..5fb23b045c92e27da10486bb54f9343bb23bc81d 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -43,6 +43,37 @@ #include "mndUser.h" #include "mndVgroup.h" +static inline int32_t mndAcquireRpc(SMnode *pMnode) { + int32_t code = 0; + taosThreadRwlockRdlock(&pMnode->lock); + if (pMnode->stopped) { + terrno = TSDB_CODE_APP_NOT_READY; + code = -1; + } else if (!mndIsLeader(pMnode)) { + code = -1; + } else { +#if 1 + atomic_add_fetch_32(&pMnode->rpcRef, 1); +#else + int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); + mTrace("mnode rpc is acquired, ref:%d", ref); +#endif + } + taosThreadRwlockUnlock(&pMnode->lock); + return code; +} + +static inline void mndReleaseRpc(SMnode *pMnode) { + taosThreadRwlockRdlock(&pMnode->lock); +#if 1 + atomic_sub_fetch_32(&pMnode->rpcRef, 1); +#else + int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1); + mTrace("mnode rpc is released, ref:%d", ref); +#endif + taosThreadRwlockUnlock(&pMnode->lock); +} + static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -201,7 +232,7 @@ static int32_t mndInitWal(SMnode *pMnode) { pMnode->pWal = walOpen(path, &cfg); if (pMnode->pWal == NULL) { - mError("failed to open wal since %s", terrstr()); + mError("failed to open wal since %s. wal:%s", terrstr(), path); return -1; } @@ -338,8 +369,9 @@ static int32_t mndExecSteps(SMnode *pMnode) { static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->msgCb = pOption->msgCb; pMnode->selfDnodeId = pOption->dnodeId; - pMnode->syncMgmt.replica = pOption->replica; - pMnode->syncMgmt.standby = pOption->standby; + pMnode->syncMgmt.selfIndex = pOption->selfIndex; + pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas; + memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas)); } SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { @@ -430,7 +462,7 @@ int32_t mndStart(SMnode *pMnode) { mError("failed to deploy sdb while start mnode"); return -1; } - mndSetRestore(pMnode, true); + mndSetRestored(pMnode, true); } grantReset(pMnode, TSDB_GRANT_ALL, 0); @@ -570,23 +602,27 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) { return 0; } - if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; + if (mndAcquireRpc(pMsg->info.node) == 0) return 0; if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { return -1; } - SEpSet epSet = {0}; - mndGetMnodeEpSet(pMsg->info.node, &epSet); + SEpSet epSet = {0}; + SMnode *pMnode = pMsg->info.node; + mndGetMnodeEpSet(pMnode, &epSet); const STraceId *trace = &pMsg->info.traceId; - mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(), - TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); + mDebug( + "msg:%p, failed to check mnode state since %s, mnode restored:%d stopped:%d, sync restored:%d role:%s type:%s " + "numOfEps:%d inUse:%d", + pMsg, terrstr(), pMnode->restored, pMnode->stopped, syncIsRestoreFinish(pMnode->syncMgmt.sync), + syncGetMyRoleStr(pMnode->syncMgmt.sync), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); if (epSet.numOfEps > 0) { for (int32_t i = 0; i < epSet.numOfEps; ++i) { - mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); } int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); @@ -633,7 +669,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); int32_t code = (*fp)(pMsg); - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mGTrace("msg:%p, won't response immediately since in progress", pMsg); @@ -669,7 +705,7 @@ int64_t mndGenerateUid(const char *name, int32_t len) { int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { - if (mndAcquireRpcRef(pMnode) != 0) return -1; + if (mndAcquireRpc(pMnode) != 0) return -1; SSdb *pSdb = pMnode->pSdb; int64_t ms = taosGetTimestampMs(); @@ -680,7 +716,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc)); if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL || pStbInfo->stbs == NULL) { - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); return -1; } @@ -800,7 +836,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pGrantInfo->timeseries_total = INT32_MAX; } - mndReleaseRpcRef(pMnode); + mndReleaseRpc(pMnode); return 0; } @@ -810,32 +846,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { return 0; } -int32_t mndAcquireRpcRef(SMnode *pMnode) { - int32_t code = 0; - taosThreadRwlockRdlock(&pMnode->lock); - if (pMnode->stopped) { - mTrace("mnode not running"); - terrno = TSDB_CODE_APP_NOT_READY; - code = -1; - } else if (!mndIsMaster(pMnode)) { - mTrace("mnode not ready, role:%s restored:%d", syncGetMyRoleStr(pMnode->syncMgmt.sync), pMnode->restored); - code = -1; - } else { - int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); - // mTrace("mnode rpc is acquired, ref:%d", ref); - } - taosThreadRwlockUnlock(&pMnode->lock); - return code; -} - -void mndReleaseRpcRef(SMnode *pMnode) { - taosThreadRwlockRdlock(&pMnode->lock); - int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1); - // mTrace("mnode rpc is released, ref:%d", ref); - taosThreadRwlockUnlock(&pMnode->lock); -} - -void mndSetRestore(SMnode *pMnode, bool restored) { +void mndSetRestored(SMnode *pMnode, bool restored) { if (restored) { taosThreadRwlockWrlock(&pMnode->lock); pMnode->restored = true; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index d0440359ffa849678b4f2f124059d11ec3214809..c27241317c2f2afa05d4744bc8e4c6e1ceb104f9 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -36,6 +36,7 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq); static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); +static void mndReloadSyncConfig(SMnode *pMnode); int32_t mndInitMnode(SMnode *pMnode) { SSdbTable table = { @@ -187,6 +188,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { } pObj->state = TAOS_SYNC_STATE_ERROR; + mndReloadSyncConfig(pSdb->pMnode); return 0; } @@ -203,6 +205,8 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) { mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); pOld->updateTime = pNew->updateTime; + mndReloadSyncConfig(pSdb->pMnode); + return 0; } @@ -233,7 +237,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { if (pIter == NULL) break; if (pObj->id == pMnode->selfDnodeId) { - if (mndIsMaster(pMnode)) { + if (mndIsLeader(pMnode)) { pEpSet->inUse = pEpSet->numOfEps; } else { pEpSet->inUse = (pEpSet->numOfEps + 1) % totalMnodes; @@ -248,6 +252,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { if (pEpSet->numOfEps == 0) { syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); } + + if (pEpSet->inUse >= pEpSet->numOfEps) { + pEpSet->inUse = 0; + } } static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { @@ -274,13 +282,72 @@ static int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnod return 0; } +static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq); + + STransAction action = { + .epSet = *pCreateEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_CREATE_MNODE, + .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + +static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq); + + STransAction action = { + .epSet = *pAlterEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_MND_ALTER_MNODE, + .acceptableCode = 0, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + +static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) { + int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, pDropReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSCreateDropMQSBNodeReq(pReq, contLen, pDropReq); + + STransAction action = { + .epSet = *pDroprEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_DROP_MNODE, + .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; int32_t numOfReplicas = 0; - SDAlterMnodeReq alterReq = {0}; SDCreateMnodeReq createReq = {0}; - SEpSet alterEpset = {0}; SEpSet createEpset = {0}; while (1) { @@ -288,75 +355,25 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); if (pIter == NULL) break; - alterReq.replicas[numOfReplicas].id = pMObj->id; - alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - if (pMObj->state == TAOS_SYNC_STATE_LEADER) { - alterEpset.inUse = numOfReplicas; - } + createReq.replicas[numOfReplicas].id = pMObj->id; + createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); numOfReplicas++; sdbRelease(pSdb, pMObj); } - alterReq.replica = numOfReplicas + 1; - alterReq.replicas[numOfReplicas].id = pDnode->id; - alterReq.replicas[numOfReplicas].port = pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.numOfEps = numOfReplicas + 1; - alterEpset.eps[numOfReplicas].port = pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - - createReq.replica = 1; - createReq.replicas[0].id = pDnode->id; - createReq.replicas[0].port = pDnode->port; - memcpy(createReq.replicas[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createReq.replica = numOfReplicas + 1; + createReq.replicas[numOfReplicas].id = pDnode->id; + createReq.replicas[numOfReplicas].port = pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createEpset.inUse = 0; createEpset.numOfEps = 1; createEpset.eps[0].port = pDnode->port; memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); - - STransAction action = { - .epSet = createEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_DND_CREATE_MNODE, - .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - - STransAction action = { - .epSet = alterEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_MND_ALTER_MNODE, - .acceptableCode = 0, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } + if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1; return 0; } @@ -374,9 +391,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, mndTransSetSerial(pTrans); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); + if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; - if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -459,107 +476,28 @@ static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeO } static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - int32_t numOfReplicas = 0; - SDAlterMnodeReq alterReq = {0}; - SDDropMnodeReq dropReq = {0}; - SSetStandbyReq standbyReq = {0}; - SEpSet alterEpset = {0}; - SEpSet dropEpSet = {0}; - - while (1) { - SMnodeObj *pMObj = NULL; - pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); - if (pIter == NULL) break; - if (pMObj->id == pObj->id) { - sdbRelease(pSdb, pMObj); - continue; - } - - alterReq.replicas[numOfReplicas].id = pMObj->id; - alterReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - - alterEpset.eps[numOfReplicas].port = pMObj->pDnode->port; - memcpy(alterEpset.eps[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); - if (pMObj->state == TAOS_SYNC_STATE_LEADER) { - alterEpset.inUse = numOfReplicas; - } - - numOfReplicas++; - sdbRelease(pSdb, pMObj); - } - - alterReq.replica = numOfReplicas; - alterEpset.numOfEps = numOfReplicas; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + int32_t numOfReplicas = 0; + SDDropMnodeReq dropReq = {0}; + SEpSet dropEpSet = {0}; dropReq.dnodeId = pDnode->id; dropEpSet.numOfEps = 1; dropEpSet.eps[0].port = pDnode->port; memcpy(dropEpSet.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - standbyReq.dnodeId = pDnode->id; - standbyReq.standby = 1; - - { - int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq); - SMsgHead *pHead = pReq; - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(MNODE_HANDLE); - - STransAction action = { - .epSet = dropEpSet, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_SYNC_SET_MNODE_STANDBY, - .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - - STransAction action = { - .epSet = alterEpset, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_MND_ALTER_MNODE, - .acceptableCode = 0, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } - - { - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void *pReq = taosMemoryMalloc(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - STransAction action = { - .epSet = dropEpSet, - .pCont = pReq, - .contLen = contLen, - .msgType = TDMT_DND_DROP_MNODE, - .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, - }; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); + if (totalMnodes == 2) { + mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes); + if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; + if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; + } else if (totalMnodes == 3) { + mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes); + if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; + if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; + } else { + return -1; } return 0; @@ -567,7 +505,6 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { if (pObj == NULL) return 0; - if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1; if (mndTransAppendNullLog(pTrans) != 0) return -1; @@ -657,7 +594,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int64_t curMs = taosGetTimestampMs(); while (numOfRows < rows) { - pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus); + pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus, true); if (pShow->pIter == NULL) break; cols = 0; @@ -712,6 +649,9 @@ static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) { } static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) { +#if 1 + return 0; +#else SMnode *pMnode = pReq->info.node; SDAlterMnodeReq alterReq = {0}; @@ -720,41 +660,107 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) { return -1; } + SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1}; + memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas)); + for (int32_t i = 0; i < option.numOfReplicas; ++i) { + if (alterReq.replicas[i].id == pMnode->selfDnodeId) { + option.selfIndex = i; + } + } + + if (option.selfIndex == -1) { + mInfo("alter mnode not processed since selfIndex is -1", terrstr()); + return 0; + } + + if (mndWriteFile(pMnode->path, &option) != 0) { + mError("failed to write mnode file since %s", terrstr()); + return -1; + } + SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1}; for (int32_t i = 0; i < alterReq.replica; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; tstrncpy(pNode->nodeFqdn, alterReq.replicas[i].fqdn, sizeof(pNode->nodeFqdn)); pNode->nodePort = alterReq.replicas[i].port; - if (alterReq.replicas[i].id == pMnode->selfDnodeId) cfg.myIndex = i; + if (alterReq.replicas[i].id == pMnode->selfDnodeId) { + cfg.myIndex = i; + } } if (cfg.myIndex == -1) { mError("failed to alter mnode since myindex is -1"); return -1; } else { - mInfo("start to alter mnode sync, replica:%d myindex:%d", cfg.replicaNum, cfg.myIndex); + mInfo("start to alter mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); for (int32_t i = 0; i < alterReq.replica; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); } } - mInfo("trans:-1, sync reconfig will be proposed"); - - SSyncMgmt *pMgmt = &pMnode->syncMgmt; - pMgmt->standby = 0; - int32_t code = syncReconfig(pMgmt->sync, &cfg); + int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); if (code != 0) { - mError("trans:-1, failed to propose sync reconfig since %s", terrstr()); - return code; + mError("failed to sync reconfig since %s", terrstr()); } else { - pMgmt->errCode = 0; - taosWLockLatch(&pMgmt->lock); - pMgmt->transId = -1; - taosWUnLockLatch(&pMgmt->lock); - tsem_wait(&pMgmt->syncSem); - mInfo("alter mnode sync result:0x%x %s", pMgmt->errCode, tstrerror(pMgmt->errCode)); - terrno = pMgmt->errCode; - return pMgmt->errCode; + mInfo("alter mnode sync success"); + } + + return code; +#endif +} + +static void mndReloadSyncConfig(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + SMnodeObj *pObj = NULL; + ESdbStatus objStatus = 0; + void *pIter = NULL; + bool hasUpdatingMnode = false; + SSyncCfg cfg = {.myIndex = -1}; + + while (1) { + pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false); + if (pIter == NULL) break; + if (objStatus == SDB_STATUS_CREATING || objStatus == SDB_STATUS_DROPPING) { + mInfo("vgId:1, has updating mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus)); + hasUpdatingMnode = true; + } + + if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) { + SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum]; + tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pObj->pDnode->port; + if (pObj->pDnode->id == pMnode->selfDnodeId) { + cfg.myIndex = cfg.replicaNum; + } + cfg.replicaNum++; + } + + sdbReleaseLock(pSdb, pObj, false); + } + + if (cfg.myIndex == -1) { + mInfo("vgId:1, mnode not reload since selfIndex is -1"); + return; + } + + if (!mndGetRestored(pMnode)) { + mInfo("vgId:1, mnode not reload since restore not finished"); + return; + } + + if (hasUpdatingMnode) { + mInfo("vgId:1, start to reload mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); + for (int32_t i = 0; i < cfg.replicaNum; ++i) { + SNodeInfo *pNode = &cfg.nodeInfo[i]; + mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); + } + + int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); + if (code != 0) { + mError("vgId:1, failed to reconfig mnode sync since %s", terrstr()); + } else { + mInfo("vgId:1, reconfig mnode sync success"); + } } } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 511a84290dd2a311f59cfc9467ff2619d92013cb..1c8ec0a302a03bd133531bccd86ee107c593ff07 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -416,7 +416,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } SStreamTask* pTask = tNewSStreamTask(pStream->uid); - if (pInnerTask == NULL) { + if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 0e4dd9fc3c51d121e54be959129cf1e98a508181..866a3fa8b9027e2c5414546ef1648c4b471cbbbb 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -107,7 +107,7 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) { if (!pMnode->deploy) { mInfo("vgId:1, sync restore finished, and will handle outstanding transactions"); mndTransPullup(pMnode); - mndSetRestore(pMnode, true); + mndSetRestored(pMnode, true); } else { mInfo("vgId:1, sync restore finished"); } @@ -225,18 +225,17 @@ int32_t mndInitSync(SMnode *pMnode) { snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP); syncInfo.pWal = pMnode->pWal; syncInfo.pFsm = mndSyncMakeFsm(pMnode); - syncInfo.isStandBy = pMgmt->standby; syncInfo.snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT; - mInfo("vgId:1, start to open sync, standby:%d", pMgmt->standby); - if (pMgmt->standby || pMgmt->replica.id > 0) { - SSyncCfg *pCfg = &syncInfo.syncCfg; - pCfg->replicaNum = 1; - pCfg->myIndex = 0; - SNodeInfo *pNode = &pCfg->nodeInfo[0]; - tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn)); - pNode->nodePort = pMgmt->replica.port; - mInfo("vgId:1, ep:%s:%u", pNode->nodeFqdn, pNode->nodePort); + mInfo("vgId:1, start to open sync, selfIndex:%d replica:%d", pMgmt->selfIndex, pMgmt->numOfReplicas); + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->replicaNum = pMgmt->numOfReplicas; + pCfg->myIndex = pMgmt->selfIndex; + for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) { + SNodeInfo *pNode = &pCfg->nodeInfo[i]; + tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pMgmt->replicas[i].port; + mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort); } tsem_init(&pMgmt->syncSem, 0, 0); @@ -250,10 +249,6 @@ int32_t mndInitSync(SMnode *pMnode) { setPingTimerMS(pMgmt->sync, 5000); setElectTimerMS(pMgmt->sync, 3000); setHeartbeatTimerMS(pMgmt->sync, 500); - /* - setElectTimerMS(pMgmt->sync, 600); - setHeartbeatTimerMS(pMgmt->sync, 300); - */ mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); return 0; @@ -319,7 +314,7 @@ void mndSyncStart(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; syncSetMsgCb(pMgmt->sync, &pMnode->msgCb); syncStart(pMgmt->sync); - mInfo("vgId:1, sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); + mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync); } void mndSyncStop(SMnode *pMnode) { @@ -331,7 +326,7 @@ void mndSyncStop(SMnode *pMnode) { taosWUnLockLatch(&pMnode->syncMgmt.lock); } -bool mndIsMaster(SMnode *pMnode) { +bool mndIsLeader(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; if (!syncIsReady(pMgmt->sync)) { @@ -340,7 +335,7 @@ bool mndIsMaster(SMnode *pMnode) { return false; } - if (!pMnode->restored) { + if (!mndGetRestored(pMnode)) { terrno = TSDB_CODE_APP_NOT_READY; return false; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d30cc8f9ca65315c0fb9f32b2148d341194c361c..a64b07fdbd6491bb16b7854758029bde06b14808 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -53,7 +53,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans); -static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); } +static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static int32_t mndProcessTransTimer(SRpcMsg *pReq); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index f922baf329fbba3c747f9be19f5bda30876d68ea..be93bd2f59da8c54c467ae90eadf6df593a087d4 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -298,6 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey); * @param pObj The object of the row. */ void sdbRelease(SSdb *pSdb, void *pObj); +void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock); /** * @brief Traverse a sdb table @@ -309,7 +310,7 @@ void sdbRelease(SSdb *pSdb, void *pObj); * @return void* The next iterator of the table. */ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj); -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status); +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock); /** * @brief Cancel a traversal diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index ecdf8c71a77d1dde73d85b2b582e626f3197c21a..0fee0fbca17685e6ee7b2a106accf87ba0d2b056 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -327,14 +327,16 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { taosThreadRwlockUnlock(pLock); } -void sdbRelease(SSdb *pSdb, void *pObj) { +void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) { if (pObj == NULL) return; SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow)); if (pRow->type >= SDB_MAX) return; TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockWrlock(pLock); + if (lock) { + taosThreadRwlockWrlock(pLock); + } int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); sdbPrintOper(pSdb, pRow, "release"); @@ -342,9 +344,13 @@ void sdbRelease(SSdb *pSdb, void *pObj) { sdbFreeRow(pSdb, pRow, true); } - taosThreadRwlockUnlock(pLock); + if (lock) { + taosThreadRwlockUnlock(pLock); + } } +void sdbRelease(SSdb *pSdb, void *pObj) { sdbReleaseLock(pSdb, pObj, true); } + void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { *ppObj = NULL; @@ -372,14 +378,16 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { return ppRow; } -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) { +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status, bool lock) { *ppObj = NULL; SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + if (lock) { + taosThreadRwlockRdlock(pLock); + } SSdbRow **ppRow = taosHashIterate(hash, pIter); while (ppRow != NULL) { @@ -395,7 +403,9 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat *status = pRow->status; break; } - taosThreadRwlockUnlock(pLock); + if (lock) { + taosThreadRwlockUnlock(pLock); + } return ppRow; } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index f96afe6fba1b081d9511a3fdd6503be73fa1bdc3..69e6cdce9f25159247ffc4b331d7a7ef701e58fa 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -115,7 +115,6 @@ typedef struct { typedef struct { SMqDataRsp dataRsp; - SMqRspHead rspHead; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; SRpcHandleInfo pInfo; } STqPushEntry; @@ -183,6 +182,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t ver); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index c8a4ff945d0cc1771b0f074b91ee55887a24fe51..1e0e75779342285143ff8713ff03c2a17480c504 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -51,7 +51,7 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema return -1; } - strcpy(pMetaRsp->tbName, tbName); + strncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); pMetaRsp->numOfColumns = pSchema->nCols; pMetaRsp->tableType = TSDB_NORMAL_TABLE; pMetaRsp->sversion = pSchema->version; @@ -693,6 +693,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (iCol >= pSchema->nCols) break; pColumn = &pSchema->pSchema[iCol]; + ASSERT(pAlterTbReq->colName); if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break; iCol++; } @@ -816,6 +817,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA const void *pData = NULL; int nData = 0; + if (pAlterTbReq->tagName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + // search name index ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 59a9f326427a64904bda2c5d86b6619587be2b47..5e1cc150637f9acd2d6ff9e60717911cd4f252f6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -171,11 +171,11 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - if (pRsp->blockNum > 0) { - ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); - } else { - ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version); - } + /*if (pRsp->blockNum > 0) {*/ + /*ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);*/ + /*} else {*/ + ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); + /*}*/ } int32_t len = 0; @@ -192,7 +192,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { return -1; } - memcpy(buf, &pPushEntry->rspHead, sizeof(SMqRspHead)); + memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); @@ -215,7 +215,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { tFormatOffset(buf1, 80, &pRsp->reqOffset); tFormatOffset(buf2, 80, &pRsp->rspOffset); tqDebug("vgId:%d, from consumer:%" PRId64 ", (epoch %d) push rsp, block num: %d, reqOffset:%s, rspOffset:%s", - TD_VID(pTq->pVnode), pPushEntry->rspHead.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); + TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); return 0; } @@ -560,9 +560,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { memcpy(pPushEntry->subKey, pHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN); dataRsp.withTbName = 0; memcpy(&pPushEntry->dataRsp, &dataRsp, sizeof(SMqDataRsp)); - pPushEntry->rspHead.consumerId = consumerId; - pPushEntry->rspHead.epoch = reqEpoch; - pPushEntry->rspHead.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + pPushEntry->dataRsp.head.consumerId = consumerId; + pPushEntry->dataRsp.head.epoch = reqEpoch; + pPushEntry->dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey) + 1, &pPushEntry, sizeof(void*)); tqDebug("tmq poll: consumer %ld, subkey %s, vg %d save handle to push mgr", consumerId, pHandle->subKey, TD_VID(pTq->pVnode)); @@ -924,7 +924,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqTableSink; + pTask->tbSink.tbSinkFunc = tqTableSink1; ASSERT(pTask->tbSink.pSchemaWrapper); ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); @@ -1010,16 +1010,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) { qError("stream task input del failed, task id %d", pTask->taskId); + + taosFreeQitem(pRefBlock); continue; } + if (streamSchedExec(pTask) < 0) { qError("stream task launch failed, task id %d", pTask->taskId); continue; } + } else { streamTaskInputFail(pTask); } } + int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index bcdac1941a9c1b94d0f67793e073b1e87218381c..4083a1a0aeef2f9a783bdace02fb89bdfeb1fe9c 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -245,7 +245,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) tqDebug("vgId:%d cannot find handle %s", pTq->pVnode->config.vgId, pPushEntry->subKey); continue; } - if (pPushEntry->dataRsp.reqOffset.version > ver) { + if (pPushEntry->dataRsp.reqOffset.version >= ver) { tqDebug("vgId:%d push entry req version %ld, while push version %ld, skip", pTq->pVnode->config.vgId, pPushEntry->dataRsp.reqOffset.version, ver); continue; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 521d12fdab18f44f63832622eb7b28934761e35c..76bfa72caa70e90c78be6ce63b1cbadd09df75ec 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -284,6 +284,212 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem return ret; } +void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper; + + int32_t blockSz = taosArrayGetSize(pBlocks); + bool createTb = true; + + SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + + tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + for (int32_t i = 0; i < blockSz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + SBatchDeleteReq deleteReq = {0}; + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + deleteReq.suid = suid; + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq); + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = serializedDeleteReq, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(serializedDeleteReq); + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + } else { + SVCreateTbReq createTbReq = {0}; + + // set const + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; + taosArrayPush(tagArray, &tagVal); + createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(tagArray); + tdDestroySVCreateTbReq(&createTbReq); + return; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + createTbReq.ctb.tagName = tagName; + + // set table name + if (pDataBlock->info.parTbName[0]) { + createTbReq.name = strdup(pDataBlock->info.parTbName); + } else { + createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + } + + int32_t schemaLen; + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + // save schema str + void* schemaStr = taosMemoryMalloc(schemaLen); + if (schemaStr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, schemaStr, schemaLen); + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); + if (code < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + tEncoderClear(&encoder); + taosMemoryFree(schemaStr); + return; + } + tEncoderClear(&encoder); + tdDestroySVCreateTbReq(&createTbReq); + + int32_t cap = sizeof(SSubmitReq); + + int32_t rows = pDataBlock->info.rows; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + + cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; + + SSubmitReq* ret = rpcMallocCont(cap); + ret->header.vgId = pVnode->config.vgId; + ret->length = sizeof(SSubmitReq); + ret->numOfBlocks = htonl(1); + + SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + + blkHead->numOfRows = htonl(pDataBlock->info.rows); + blkHead->sversion = htonl(pTSchema->version); + blkHead->suid = htobe64(suid); + // uid is assigned by vnode + blkHead->uid = 0; + blkHead->schemaLen = 0; + + tqDebug("tq sink, convert block %d, rows: %d", i, rows); + + int32_t dataLen = 0; + void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); + STSRow* rowData = blkSchema; + if (createTb) { + memcpy(blkSchema, schemaStr, schemaLen); + blkHead->schemaLen = htonl(schemaLen); + rowData = POINTER_SHIFT(blkSchema, schemaLen); + } + + taosMemoryFree(schemaStr); + + for (int32_t j = 0; j < rows; j++) { + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTSchema->version); + tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); + tdSRowResetBuf(&rb, rowData); + + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pColumn = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); + } else { + void* colData = colDataGetData(pColData, j); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); + } + } + tdSRowEnd(&rb); + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + dataLen += rowLen; + } + blkHead->dataLen = htonl(dataLen); + + ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; + ret->length = htonl(ret->length); + + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = ret, + .contLen = ntohl(ret->length), + }; + + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(ret); + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + } + taosArrayDestroy(tagArray); +} + void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ac6be9af2d85fa15387d8fd8b1c760055fe061c0..0ee2e310970179f89229836c2d66735fa3083943 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -378,10 +378,10 @@ static int32_t getTableDelSkyline(STbData *pMem, STbData *pIMem, SDelFReader *pD if (code) goto _err; } +_err: if (aDelData) { taosArrayDestroy(aDelData); } -_err: return code; } @@ -399,14 +399,13 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx); SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ); - if (code) goto _err; *pDelIdx = *pIdx; +_err: if (pDelIdxArray) { taosArrayDestroy(pDelIdxArray); } -_err: return code; } @@ -429,7 +428,8 @@ typedef struct { SDataFReader *pDataFReader; TSDBROW row; - SMergeTree mergeTree; + SMergeTree mergeTree; + SMergeTree *pMergeTree; } SFSLastNextRowIter; static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { @@ -444,11 +444,14 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { case SFSLASTNEXTROW_FILESET: { SDFileSet *pFileSet = NULL; _next_fileset: + if (state->pMergeTree != NULL) { + tMergeTreeClose(state->pMergeTree); + state->pMergeTree = NULL; + } + if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - tMergeTreeClose(&state->mergeTree); - *ppRow = NULL; return code; } @@ -460,10 +463,10 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, pLoadInfo, true, NULL); + state->pMergeTree = &state->mergeTree; bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; - tMergeTreeClose(&state->mergeTree); goto _next_fileset; } state->state = SFSLASTNEXTROW_BLOCKROW; @@ -475,6 +478,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; } + return code; default: ASSERT(0); @@ -486,6 +490,11 @@ _err: tsdbDataFReaderClose(&state->pDataFReader); state->pDataFReader = NULL; } + if (state->pMergeTree != NULL) { + tMergeTreeClose(state->pMergeTree); + state->pMergeTree = NULL; + } + *ppRow = NULL; return code; @@ -504,6 +513,11 @@ int32_t clearNextRowFromFSLast(void *iter) { state->pDataFReader = NULL; } + if (state->pMergeTree != NULL) { + tMergeTreeClose(state->pMergeTree); + state->pMergeTree = NULL; + } + return code; } @@ -584,8 +598,6 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { * &state->blockIdx); */ state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); - if (code) goto _err; - if (!state->pBlockIdx) { goto _next_fileset; } @@ -883,10 +895,16 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs if (code) goto _err; code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); - if (code) goto _err; + if (code) { + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline); - if (code) goto _err; + if (code) { + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } tsdbDelFReaderClose(&pDelFReader); } else { @@ -1216,6 +1234,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { _err: nextRowIterClose(&iter); taosMemoryFreeClear(pTSchema); + *ppLastArray = NULL; + taosArrayDestroy(pColArray); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 995d7a315550aa695b5f3bab816815edaf370800..a4d993bc6c318967d2624aed82ce66918a778d00 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -396,12 +396,19 @@ _exit: return code; } +static int32_t tDataIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { + SDataIter *pIter1 = (SDataIter *)((uint8_t *)n1 - offsetof(SDataIter, n)); + SDataIter *pIter2 = (SDataIter *)((uint8_t *)n2 - offsetof(SDataIter, n)); + + return tRowInfoCmprFn(&pIter1->r, &pIter2->r); +} + static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { int32_t code = 0; int32_t lino = 0; pCommitter->pIter = NULL; - tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); + tRBTreeCreate(&pCommitter->rbt, tDataIterCmprFn); // memory TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN}; @@ -1038,7 +1045,9 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; - ASSERT(eno == 0); + ASSERT(eno == 0 && + "tsdbCommit failure" + "Restart taosd"); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 85514ed5b607814ef441ad239724e045002c44b8..6fd562959229fd4c75b4f333f118caa86a66555c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -610,9 +610,6 @@ int32_t tsdbFSRollback(STsdbFS *pFS) { ASSERT(0); return code; - -_err: - return code; } int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile) { @@ -822,7 +819,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(fSet.pHeadF); } } else { @@ -866,7 +863,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&fSet.pSmaF->nRef, 1); if (nRef == 0) { tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pSmaF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(fSet.pSmaF); } } else { @@ -877,7 +874,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { // stt if (sameDisk) { if (pSetNew->nSttF > pSetOld->nSttF) { - ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1); + ASSERT(pSetNew->nSttF == pSetOld->nSttF + 1); pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -964,7 +961,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pSetOld->pHeadF); } @@ -1104,7 +1101,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { ASSERT(nRef >= 0); if (nRef == 0) { tsdbDelFileName(pTsdb, pFS->pDelFile, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pFS->pDelFile); } } @@ -1117,7 +1114,7 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { ASSERT(nRef >= 0); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(pSet->pHeadF); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index a6628463f8ff231505052dc3a48d8c7a59a6eaa5..0189ced3c6819f8b4d11adf63bf977579f5efc56 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -160,6 +160,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid goto _err; } + ASSERT(pPool != NULL); // do delete SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData)); if (pDelData == NULL) { @@ -353,6 +354,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; + ASSERT(pPool != NULL); pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); if (pTbData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -492,6 +494,7 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN // node level = tsdbMemSkipListRandLevel(&pTbData->sl); + ASSERT(pPool != NULL); pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow)); if (pNode == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -559,6 +562,8 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i // backward put first data row.pTSRow = tGetSubmitBlkNext(&blkIter); + if (row.pTSRow == NULL) return code; + key.ts = row.pTSRow->ts; nRow++; tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 4d54fbbd797f7ca9984664866f7db772d19473fd..2f4fdfc5f8882d6b69425d0f9c14853424c36eb3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -101,8 +101,8 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { int32_t code = 0; - SSttBlockLoadInfo* pInfo = pIter->pBlockLoadInfo; - if (pInfo->blockIndex[0] == pIter->iSttBlk) { + SSttBlockLoadInfo *pInfo = pIter->pBlockLoadInfo; + if (pInfo->blockIndex[0] == pIter->iSttBlk) { if (pInfo->currentLoadBlockIndex != 0) { tsdbDebug("current load index is set to 0, block index:%d, file index:%d, due to uid:%" PRIu64 ", load data, %s", pIter->iSttBlk, pIter->iStt, pIter->uid, idStr); @@ -113,7 +113,7 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { if (pInfo->blockIndex[1] == pIter->iSttBlk) { if (pInfo->currentLoadBlockIndex != 1) { - tsdbDebug("current load index is set to 1, block index:%d, file index:%d, due to uid:%"PRIu64", load data, %s", + tsdbDebug("current load index is set to 1, block index:%d, file index:%d, due to uid:%" PRIu64 ", load data, %s", pIter->iSttBlk, pIter->iStt, pIter->uid, idStr); pInfo->currentLoadBlockIndex = 1; } @@ -133,18 +133,24 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { id.uid = pIter->uid; } - tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } double el = (taosGetTimestampUs() - st) / 1000.0; pInfo->elapsedTime += el; pInfo->loadBlocks += 1; - tsdbDebug("read last block, total load:%d, trigger by uid:%"PRIu64", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s", - pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el, idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + tsdbDebug("read last block, total load:%d, trigger by uid:%" PRIu64 + ", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s", + pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el, + idStr); pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; tsdbDebug("last block index list:%d, %d, %s", pInfo->blockIndex[0], pInfo->blockIndex[1], idStr); @@ -336,7 +342,7 @@ _exit: void tLDataIterClose(SLDataIter *pIter) { taosMemoryFree(pIter); } -void tLDataIterNextBlock(SLDataIter *pIter, const char* idStr) { +void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) { int32_t step = pIter->backward ? -1 : 1; int32_t oldIndex = pIter->iSttBlk; @@ -386,10 +392,10 @@ void tLDataIterNextBlock(SLDataIter *pIter, const char* idStr) { if (index != -1) { pIter->iSttBlk = index; pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk); - tsdbDebug("try next last file block:%d from %d, trigger by uid:%"PRIu64", file index:%d, %s", pIter->iSttBlk, oldIndex, pIter->uid, pIter->iStt, - idStr); + tsdbDebug("try next last file block:%d from %d, trigger by uid:%" PRIu64 ", file index:%d, %s", pIter->iSttBlk, + oldIndex, pIter->uid, pIter->iStt, idStr); } else { - tsdbDebug("no more last block qualified, uid:%"PRIu64", file index::%d, %s", pIter->uid, oldIndex, idStr); + tsdbDebug("no more last block qualified, uid:%" PRIu64 ", file index::%d, %s", pIter->uid, oldIndex, idStr); } } @@ -457,8 +463,8 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { } bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { - int32_t code = 0; int32_t step = pIter->backward ? -1 : 1; + terrno = TSDB_CODE_SUCCESS; // no qualified last file block in current file, no need to fetch row if (pIter->pSttBlk == NULL) { @@ -467,6 +473,10 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { int32_t iBlockL = pIter->iSttBlk; SBlockData *pBlockData = loadLastBlock(pIter, idStr); + if (pBlockData == NULL && terrno != TSDB_CODE_SUCCESS) { + goto _exit; + } + pIter->iRow += step; while (1) { @@ -492,19 +502,15 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow); _exit: - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - } - - return (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); + return (terrno == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); } SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } // SMergeTree ================================================= -static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { - SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - sizeof(SRBTreeNode)); - SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - sizeof(SRBTreeNode)); +static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTreeNode *p2) { + SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node)); + SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node)); TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row); TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row); @@ -541,6 +547,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead pMTree->pLoadInfo = pBlockLoadInfo; pMTree->destroyLoadInfo = destroyLoadInfo; + ASSERT(pMTree->pLoadInfo != NULL); for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file struct SLDataIter *pIter = NULL; @@ -581,7 +588,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { // compare with min in RB Tree pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); if (pMTree->pIter && pIter) { - int32_t c = pMTree->rbt.cmprFn(RBTREE_NODE_PAYLOAD(&pMTree->pIter->node), RBTREE_NODE_PAYLOAD(&pIter->node)); + int32_t c = pMTree->rbt.cmprFn(&pMTree->pIter->node, &pIter->node); if (c > 0) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); pMTree->pIter = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index e4080ccf1e36bd04032d0c16fe11b094c3b69d31..51978234901a6a4e26f78d0c7d867818d64fa7eb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -87,9 +87,13 @@ _err: int tsdbClose(STsdb **pTsdb) { if (*pTsdb) { - taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + taosThreadRwlockWrlock(&(*pTsdb)->rwLock); tsdbMemTableDestroy((*pTsdb)->mem); (*pTsdb)->mem = NULL; + taosThreadRwlockUnlock(&(*pTsdb)->rwLock); + + taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + tsdbFSClose(*pTsdb); tsdbCloseCache(*pTsdb); taosMemoryFreeClear(*pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4bb1f2d2fe6c8cef477118eb0f33ce32999720d2..670691ab1fff940ed030842c6003d17501958322 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -340,7 +340,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); if (pIter->pLastBlockReader == NULL) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr); + tsdbError("failed to prepare the last block iterator, code:%s %s", tstrerror(code), pReader->idStr); return code; } } @@ -646,7 +646,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug( - "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " + "load block of %"PRIzu" tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " "time:%.2f ms %s", numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, pReader->idStr); @@ -1515,6 +1515,11 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* taosMemoryFree(pReader->pMemSchema); int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + return pReader->pMemSchema; } @@ -1645,7 +1650,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STSRow* pTSRow = NULL; SRowMerger merge = {0}; TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbTrace("fRow ptr:%p, %d, uid:%"PRIu64", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { @@ -2676,9 +2681,9 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret int8_t level = 0; int8_t precision = pVnode->config.tsdbCfg.precision; int64_t now = taosGetTimestamp(precision); - int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1 - : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000 - : 1000000); + int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L + : 1000000L); for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + level; @@ -3085,7 +3090,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, piRow); - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); } int32_t code = tRowMergerGetRow(&merge, pTSRow); @@ -3372,14 +3377,13 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl if (pCond->suid != 0) { pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1, 1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:%" PRId64 " , %s", pReader->suid, -1, - pReader->idStr); + tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr); } } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:%" PRId64 " , %s", pKey->uid, -1, pReader->idStr); + tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr); } } @@ -3395,19 +3399,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl goto _err; } - code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { - code = doOpenReaderImpl(pReader); + if (numOfTables > 0) { + code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { - return code; + goto _err; } - } else { - STsdbReader* pPrevReader = pReader->innerReader[0]; - STsdbReader* pNextReader = pReader->innerReader[1]; + + if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + STsdbReader* pPrevReader = pReader->innerReader[0]; + STsdbReader* pNextReader = pReader->innerReader[1]; // we need only one row pPrevReader->capacity = 1; @@ -3422,19 +3427,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl pNextReader->pMemSchema = pReader->pMemSchema; pNextReader->pReadSnap = pReader->pReadSnap; - code = doOpenReaderImpl(pPrevReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doOpenReaderImpl(pPrevReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - code = doOpenReaderImpl(pNextReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = doOpenReaderImpl(pNextReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - code = doOpenReaderImpl(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } @@ -3442,7 +3448,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl return code; _err: - tsdbError("failed to create data reader, code:%s %s", tstrerror(code), pReader->idStr); + tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); return code; } @@ -3513,6 +3519,10 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pLReader); } + if (pReader->innerReader[0] != 0) { + tsdbUntakeReadSnap(pReader->innerReader[0]->pTsdb, pReader->innerReader[0]->pReadSnap, pReader->idStr); + } + tsdbDebug( "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 " SMA-time:%.2f ms, fileBlocks:%" PRId64 @@ -3727,7 +3737,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { } int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { - if (isEmptyQueryTimeWindow(&pReader->window)) { + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) { return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 167b4a104c84d73acb4cde1bf192440645adbf31..84b3afe72c70147a28bdb55446c806a1f1ebd70e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -18,7 +18,7 @@ // =============== PAGE-WISE FILE =============== static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { int32_t code = 0; - STsdbFD *pFD; + STsdbFD *pFD = NULL; *ppFD = NULL; @@ -35,6 +35,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd pFD->pFD = taosOpenFile(path, flag); if (pFD->pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pFD); goto _exit; } pFD->szPage = szPage; @@ -42,11 +43,15 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd pFD->pBuf = taosMemoryCalloc(1, szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + taosCloseFile(&pFD->pFD); taosMemoryFree(pFD); goto _exit; } if (taosStatFile(path, &pFD->szFile, NULL) < 0) { code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); + taosMemoryFree(pFD); goto _exit; } ASSERT(pFD->szFile % szPage == 0); @@ -59,10 +64,12 @@ _exit: static void tsdbCloseFile(STsdbFD **ppFD) { STsdbFD *pFD = *ppFD; - taosMemoryFree(pFD->pBuf); - taosCloseFile(&pFD->pFD); - taosMemoryFree(pFD); - *ppFD = NULL; + if (pFD) { + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); + taosMemoryFree(pFD); + *ppFD = NULL; + } } static int32_t tsdbWriteFilePage(STsdbFD *pFD) { @@ -117,7 +124,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { } // check - if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { + if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { code = TSDB_CODE_FILE_CORRUPTED; goto _exit; } @@ -443,7 +450,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p pBlockIdx->size = size; pHeadFile->size += size; - tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 + tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%" PRId64 " suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 " size:%" PRId64 " nItem:%d", TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem); @@ -457,7 +464,7 @@ _err: int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) { int32_t code = 0; SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1]; - int64_t size; + int64_t size = 0; int64_t n; // check @@ -836,7 +843,8 @@ _err: // SDataFReader ==================================================== int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t code = 0; - SDataFReader *pReader; + int32_t lino = 0; + SDataFReader *pReader = NULL; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; char fname[TSDB_FILENAME_LEN]; @@ -844,7 +852,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->pTsdb = pTsdb; pReader->pSet = pSet; @@ -852,31 +860,40 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS // head tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // data tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // sma tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // stt for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - *ppReader = pReader; - return code; - -_err: - tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; +_exit: + if (code) { + *ppReader = NULL; + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + + if (pReader) { + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) tsdbCloseFile(&pReader->aSttFD[iStt]); + tsdbCloseFile(&pReader->pSmaFD); + tsdbCloseFile(&pReader->pDataFD); + tsdbCloseFile(&pReader->pHeadFD); + taosMemoryFree(pReader); + } + } else { + *ppReader = pReader; + } return code; } @@ -906,10 +923,6 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { taosMemoryFree(*ppReader); *ppReader = NULL; return code; - -_err: - tsdbError("vgId:%d, data file reader close failed since %s", TD_VID((*ppReader)->pTsdb->pVnode), tstrerror(code)); - return code; } int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { @@ -1289,16 +1302,17 @@ _exit: // SDelFWriter ==================================================== int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; char fname[TSDB_FILENAME_LEN]; uint8_t hdr[TSDB_FHDR_SIZE] = {0}; - SDelFWriter *pDelFWriter; + SDelFWriter *pDelFWriter = NULL; int64_t n; // alloc pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); if (pDelFWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pDelFWriter->pTsdb = pTsdb; pDelFWriter->fDel = *pFile; @@ -1306,21 +1320,28 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // update header code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pDelFWriter->fDel.size = TSDB_FHDR_SIZE; pDelFWriter->fDel.offset = 0; *ppWriter = pDelFWriter; - return code; -_err: - tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; +_exit: + if (code) { + if (pDelFWriter) { + taosMemoryFree(pDelFWriter); + tsdbCloseFile(&pDelFWriter->pWriteH); + } + *ppWriter = NULL; + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(errno)); + } else { + *ppWriter = pDelFWriter; + } return code; } @@ -1456,15 +1477,15 @@ struct SDelFReader { int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; char fname[TSDB_FILENAME_LEN]; - SDelFReader *pDelFReader; - int64_t n; + SDelFReader *pDelFReader = NULL; // alloc pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); if (pDelFReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + goto _exit; } // open impl @@ -1473,14 +1494,18 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ, &pDelFReader->pReadH); - if (code) goto _err; - - *ppReader = pDelFReader; - return code; + if (code) { + taosMemoryFree(pDelFReader); + goto _exit; + } -_err: - tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; +_exit: + if (code) { + *ppReader = NULL; + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + *ppReader = pDelFReader; + } return code; } @@ -1573,4 +1598,4 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { _err: tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a928dc3484c67461a8b5a49c39d60e3af4336e8b..74c704598d891d5c50e0fd17766834401891e96d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -67,6 +67,13 @@ extern int32_t tRowInfoCmprFn(const void* p1, const void* p2); extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData); extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo); +static int32_t tFDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pNode2) { + SFDataIter* pIter1 = (SFDataIter*)(((uint8_t*)pNode1) - offsetof(SFDataIter, n)); + SFDataIter* pIter2 = (SFDataIter*)(((uint8_t*)pNode2) - offsetof(SFDataIter, n)); + + return tRowInfoCmprFn(&pIter1->rInfo, &pIter2->rInfo); +} + static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { int32_t code = 0; @@ -79,7 +86,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { if (code) goto _err; pReader->pIter = NULL; - tRBTreeCreate(&pReader->rbt, tRowInfoCmprFn); + tRBTreeCreate(&pReader->rbt, tFDataIterCmprFn); // .data file SFDataIter* pIter = &pReader->aFDataIter[0]; @@ -421,7 +428,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { n += tPutDelData((*ppData) + n, pDelData); } - tsdbInfo("vgId:%d, vnode snapshot tsdb read del data for %s, suid:%" PRId64 " uid:%d" PRId64 " size:%d", + tsdbInfo("vgId:%d, vnode snapshot tsdb read del data for %s, suid:%" PRId64 " uid:%" PRId64 " size:%d", TD_VID(pTsdb->pVnode), pTsdb->path, pDelIdx->suid, pDelIdx->uid, size); break; @@ -431,20 +438,21 @@ _exit: return code; _err: - tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->pVnode, + tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); return code; } int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) { int32_t code = 0; + int32_t lino = 0; STsdbSnapReader* pReader = NULL; // alloc pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->pTsdb = pTsdb; pReader->sver = sver; @@ -454,19 +462,19 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type code = taosThreadRwlockRdlock(&pTsdb->rwLock); if (code) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbFSRef(pTsdb, &pReader->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = taosThreadRwlockUnlock(&pTsdb->rwLock); if (code) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // data @@ -478,43 +486,52 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pIter->aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } else { pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pIter->aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } code = tBlockDataCreate(&pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pReader->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // del pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); if (pReader->aDelIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pReader->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path); - *ppReader = pReader; - return code; - -_err: - tsdbError("vgId:%d, vnode snapshot tsdb reader open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); - *ppReader = NULL; +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, TSDB path: %s", TD_VID(pTsdb->pVnode), lino, tstrerror(code), + pTsdb->path); + *ppReader = NULL; + + if (pReader) { + taosArrayDestroy(pReader->aDelData); + taosArrayDestroy(pReader->aDelIdx); + tBlockDataDestroy(&pReader->bData, 1); + tsdbFSDestroy(&pReader->fs); + taosMemoryFree(pReader); + } + } else { + *ppReader = pReader; + tsdbInfo("vgId:%d, vnode snapshot tsdb reader opened for %s", TD_VID(pTsdb->pVnode), pTsdb->path); + } return code; } @@ -1247,20 +1264,21 @@ _err: // APIs int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { int32_t code = 0; + int32_t lino = 0; STsdbSnapWriter* pWriter = NULL; // alloc pWriter = (STsdbSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); if (pWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->pTsdb = pTsdb; pWriter->sver = sver; pWriter->ever = ever; code = tsdbFSCopy(pTsdb, &pWriter->fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // config pWriter->minutes = pTsdb->keepCfg.days; @@ -1272,7 +1290,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr // SNAP_DATA_TSDB code = tBlockDataCreate(&pWriter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pWriter->fid = INT32_MIN; pWriter->id = (TABLEID){0}; @@ -1280,53 +1298,67 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pWriter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pWriter->dReader.bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // Writer pWriter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pWriter->dWriter.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pWriter->dWriter.aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pWriter->dWriter.bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataCreate(&pWriter->dWriter.sData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // SNAP_DATA_DEL pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxR == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pWriter->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->aDelIdxW = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxW == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - *ppWriter = pWriter; - - tsdbInfo("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); - return code; - -_err: - tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); - *ppWriter = NULL; +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + *ppWriter = NULL; + + if (pWriter) { + if (pWriter->aDelIdxW) taosArrayDestroy(pWriter->aDelIdxW); + if (pWriter->aDelData) taosArrayDestroy(pWriter->aDelData); + if (pWriter->aDelIdxR) taosArrayDestroy(pWriter->aDelIdxR); + tBlockDataDestroy(&pWriter->dWriter.sData, 1); + tBlockDataDestroy(&pWriter->dWriter.bData, 1); + if (pWriter->dWriter.aSttBlk) taosArrayDestroy(pWriter->dWriter.aSttBlk); + if (pWriter->dWriter.aBlockIdx) taosArrayDestroy(pWriter->dWriter.aBlockIdx); + tBlockDataDestroy(&pWriter->dReader.bData, 1); + if (pWriter->dReader.aBlockIdx) taosArrayDestroy(pWriter->dReader.aBlockIdx); + tBlockDataDestroy(&pWriter->bData, 1); + tsdbFSDestroy(&pWriter->fs); + taosMemoryFree(pWriter); + } + } else { + tsdbDebug("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); + *ppWriter = pWriter; + } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 0a9fbf92a4bf62326aa9755b827b83d0d510d2f7..f38802aee78021c158a29fed44e0fc14e06fc482 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -34,7 +34,9 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * } // loop to insert - tInitSubmitMsgIter(pMsg, &msgIter); + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { + return -1; + } while (true) { SSubmitBlkRsp r = {0}; tGetSubmitMsgNext(&msgIter, &pBlock); diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 3c0a041197d372f2540e11e8edb162b15c39c2e7..682f2b4225d6f8151a0061a1a420eca753ede3c3 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -112,6 +112,8 @@ void vnodeBufPoolReset(SVBufPool *pPool) { void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { SVBufPoolNode *pNode; void *p = NULL; + ASSERT(pPool != NULL); + taosThreadSpinLock(&pPool->lock); if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node @@ -141,17 +143,17 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { } void vnodeBufPoolFree(SVBufPool *pPool, void *p) { - uint8_t *ptr = (uint8_t *)p; - SVBufPoolNode *pNode; + // uint8_t *ptr = (uint8_t *)p; + // SVBufPoolNode *pNode; - if (ptr < pPool->node.data || ptr >= pPool->node.data + pPool->node.size) { - pNode = &((SVBufPoolNode *)p)[-1]; - *pNode->pnext = pNode->prev; - pNode->prev->pnext = pNode->pnext; + // if (ptr < pPool->node.data || ptr >= pPool->node.data + pPool->node.size) { + // pNode = &((SVBufPoolNode *)p)[-1]; + // *pNode->pnext = pNode->prev; + // pNode->prev->pnext = pNode->pnext; - pPool->size = pPool->size - sizeof(*pNode) - pNode->size; - taosMemoryFree(pNode); - } + // pPool->size = pPool->size - sizeof(*pNode) - pNode->size; + // taosMemoryFree(pNode); + // } } void vnodeBufPoolRef(SVBufPool *pPool) { diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 07d9b9626137bf7a80f049367fa9f694cbf127fd..07c4c32955ece956b44beaba5ca78bcd0877e916 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -73,7 +73,7 @@ int vnodeBegin(SVnode *pVnode) { int vnodeShouldCommit(SVnode *pVnode) { if (pVnode->inUse) { - return pVnode->inUse->size > pVnode->inUse->node.size; + return osDataSpaceAvailable() && (pVnode->inUse->size > pVnode->inUse->node.size); } return false; } @@ -89,6 +89,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { data = NULL; if (vnodeEncodeInfo(pInfo, &data) < 0) { + vError("failed to encode json info."); return -1; } @@ -101,7 +102,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { } if (taosWriteFile(pFile, data, strlen(data)) < 0) { - vError("failed to write info file:%s data:%s", fname, terrstr()); + vError("failed to write info file:%s error:%s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -233,15 +234,15 @@ int vnodeCommit(SVnode *pVnode) { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } if (vnodeSaveInfo(dir, &info) < 0) { - ASSERT(0); + vError("vgId:%d, failed to save vnode info since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } walBeginSnapshot(pVnode->pWal, pVnode->state.applied); // preCommit // smaSyncPreCommit(pVnode->pSma); - if (smaAsyncPreCommit(pVnode->pSma) < 0) { - ASSERT(0); + if(smaAsyncPreCommit(pVnode->pSma) < 0){ + vError("vgId:%d, failed to async pre-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -250,44 +251,44 @@ int vnodeCommit(SVnode *pVnode) { // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (VND_IS_RSMA(pVnode)) { if (smaAsyncCommit(pVnode->pSma) < 0) { - ASSERT(0); + vError("vgId:%d, failed to async commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA1(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA2(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } else { if (tsdbCommit(pVnode->pTsdb) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } if (tqCommit(pVnode->pTq) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tq since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // walCommit (TODO) // commit info if (vnodeCommitInfo(dir, &info) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit vnode info since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -296,7 +297,7 @@ int vnodeCommit(SVnode *pVnode) { // postCommit // smaSyncPostCommit(pVnode->pSma); if (smaAsyncPostCommit(pVnode->pSma) < 0) { - ASSERT(0); + vError("vgId:%d, failed to async post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 314c0e7390a6616e8bceba613cf4a57687db8fef..9fe37505add7d2ec768233637f9da2c55f7124fb 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -46,11 +46,17 @@ int vnodeInit(int nthreads) { return 0; } - vnodeGlobal.stop = 0; + taosThreadMutexInit(&vnodeGlobal.mutex, NULL); + taosThreadCondInit(&vnodeGlobal.hasTask, NULL); + taosThreadMutexLock(&vnodeGlobal.mutex); + + vnodeGlobal.stop = 0; vnodeGlobal.queue.next = &vnodeGlobal.queue; vnodeGlobal.queue.prev = &vnodeGlobal.queue; + taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + vnodeGlobal.nthreads = nthreads; vnodeGlobal.threads = taosMemoryCalloc(nthreads, sizeof(TdThread)); if (vnodeGlobal.threads == NULL) { @@ -59,9 +65,6 @@ int vnodeInit(int nthreads) { return -1; } - taosThreadMutexInit(&vnodeGlobal.mutex, NULL); - taosThreadCondInit(&vnodeGlobal.hasTask, NULL); - for (int i = 0; i < nthreads; i++) { taosThreadCreate(&(vnodeGlobal.threads[i]), NULL, loop, NULL); } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index de3af7cde6933d1bce189f3b3c8ba81b52293056..001bb5f7f209f14887d637103574c9e79ab5878d 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -38,7 +38,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { if (taosMkDir(path)) { return TAOS_SYSTEM_ERROR(errno); } - strcpy(dir, path); + snprintf(dir, TSDB_FILENAME_LEN, "%s", path); } if (pCfg) { @@ -51,7 +51,7 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { info.state.commitID = 0; if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { - vError("vgId:%d, failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno)); return -1; } @@ -140,7 +140,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { - vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode wal since %s. wal:%s", TD_VID(pVnode), tstrerror(terrno), tdir); goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 08c3a34699b6f9f83b366e42f77e42a6094f09ed..9cbb1b8af491ea1567c8d9a3ada47614749503bc 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -166,7 +166,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (*ppData) { goto _exit; } else { - pReader->tqHandleDone = 1; + pReader->tqOffsetDone = 1; code = tqOffsetReaderClose(&pReader->pTqOffsetReader); if (code) goto _err; } @@ -219,7 +219,7 @@ _exit: return code; _err: - vError("vgId:% vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); + vError("vgId:%d vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); return code; } @@ -260,7 +260,10 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr // commit it code = vnodeCommit(pVnode); - if (code) goto _err; + if (code) { + taosMemoryFree(pWriter); + goto _err; + } // inc commit ID pVnode->state.commitID++; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 131327976f06d7a610a2319b1f4a40e2c097c51f..3556b47dd3c88b65517ad4706618917e00bcd3d6 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -41,16 +41,28 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t nReqs; tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); - tStartDecode(&dc); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } - tDecodeI32v(&dc, &nReqs); + if (tDecodeI32v(&dc, &nReqs) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } for (int32_t iReq = 0; iReq < nReqs; iReq++) { tb_uid_t uid = tGenIdPI64(); char *name = NULL; tStartDecode(&dc); - tDecodeI32v(&dc, NULL); - tDecodeCStr(&dc, &name); + if (tDecodeI32v(&dc, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + if (tDecodeCStr(&dc, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } *(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos + 8) = ctime; @@ -68,7 +80,10 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int64_t ctime = taosGetTimestampMs(); tb_uid_t uid; - tInitSubmitMsgIter(pSubmitReq, &msgIter); + if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { + code = terrno; + goto _err; + } for (;;) { tGetSubmitMsgNext(&msgIter, &pBlock); @@ -78,10 +93,19 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { char *name = NULL; tDecoderInit(&dc, pBlock->data, msgIter.schemaLen); - tStartDecode(&dc); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } - tDecodeI32v(&dc, NULL); - tDecodeCStr(&dc, &name); + if (tDecodeI32v(&dc, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + if (tDecodeCStr(&dc, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } uid = metaGetTableEntryUidByName(pVnode->pMeta, name); if (uid == 0) { @@ -145,6 +169,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t len; int32_t ret; + if (!pVnode->inUse) { + terrno = TSDB_CODE_VND_NOT_SYNCED; + vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); + return -1; + } + vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); @@ -265,10 +295,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp _do_commit: vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change - vnodeCommit(pVnode); + if (vnodeCommit(pVnode) < 0) { + vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } // start a new one - vnodeBegin(pVnode); + if (vnodeBegin(pVnode) < 0) { + vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } } return 0; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 7b8c66e36889dde2fe0c624c6adb099151455501..744d6eab0693293df5887e613539cc9935beba6d 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -638,7 +638,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_QNODE_LIST; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pConn->mgmtEps.inUse); @@ -692,7 +692,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_DNODE_LIST; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get dnode list from mnode, mgmtEpInUse:%d", pConn->mgmtEps.inUse); @@ -743,7 +743,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU int32_t msgLen = 0; int32_t reqType = TDMT_MND_USE_DB; SCtgTask* pTask = tReq ? tReq->pTask : NULL; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); @@ -795,7 +795,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_DB_CFG; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName); @@ -850,7 +850,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_INDEX; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get index from mnode, indexName:%s", indexName); @@ -905,7 +905,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_TABLE_INDEX; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(name, tbFName); @@ -962,7 +962,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_RETRIEVE_FUNC; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get udf info from mnode, funcName:%s", funcName); @@ -1017,7 +1017,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_USER_AUTH; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get user auth from mnode, user:%s", user); @@ -1077,7 +1077,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* int32_t reqType = TDMT_MND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFName, "%s.%s", dbFName, tbName); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName); @@ -1140,7 +1140,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa int32_t reqType = TDMT_VND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; ctgDebug("try to get table meta from vnode, vgId:%d, ep num:%d, ep %s:%d, tbFName:%s", vgroupInfo->vgId, @@ -1209,7 +1209,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t reqType = TDMT_VND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFName); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; @@ -1274,7 +1274,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t reqType = TDMT_MND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFName); - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; @@ -1326,7 +1326,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_SERVER_VERSION; - void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + void* (*mallocFp)(int64_t) = pTask ? taosMemoryMalloc : rpcMallocCont; qDebug("try to get svr ver from mnode"); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f9e47f92fecaea8e7cbf5fa0b4cecaa2118c7971..fc996a60033278cc8460c187f73d58e7b7cda441 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -212,6 +212,7 @@ typedef struct SExprSupp { int32_t numOfExprs; // the number of scalar expression in group operator SqlFunctionCtx* pCtx; int32_t* rowEntryInfoOffset; // offset value for each row result cell info + SFilterInfo* pFilterInfo; } SExprSupp; typedef struct SOperatorInfo { @@ -479,6 +480,7 @@ typedef struct SStreamScanInfo { SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; SExprSupp tbnameCalSup; + SExprSupp tagCalSup; int32_t primaryTsIndex; // primary time stamp slot id SReadHandle readHandle; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. @@ -775,6 +777,7 @@ typedef struct SStreamPartitionOperatorInfo { SPartitionBySupporter partitionSup; SExprSupp scalarSup; SExprSupp tbnameCalSup; + SExprSupp tagCalSup; SHashObj* pPartitions; void* parIte; SSDataBlock* pInputDataBlock; @@ -924,7 +927,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); void doSetOperatorCompleted(SOperatorInfo* pOperator); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock, const char* idStr); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 6a43ea40c69e491f93c3a83f4394768beb86e8a0..ce0ff485ab41e97ac032b987e4558fec2d0c1f4a 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -81,7 +81,7 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { } pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows; - qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows, + qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows); tFreeSSubmitRsp(pInserter->submitRes.pRsp); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2f6b737f79fcad308f5247de7487c8a9cb42dbf2..f99c83238198f23586707b8e09ddb5ff4f2da1f4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1138,7 +1138,7 @@ static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, i s.bytes = bytes; s.slotId = slotId; s.precision = precision; - strncpy(s.name, name, tListLen(s.name)); + tstrncpy(s.name, name, tListLen(s.name)); return s; } @@ -1201,14 +1201,21 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName); - pExp->pExpr->_function.functionId = pFuncNode->funcId; - pExp->pExpr->_function.pFunctNode = pFuncNode; + tExprNode* pExprNode = pExp->pExpr; + + pExprNode->_function.functionId = pFuncNode->funcId; + pExprNode->_function.pFunctNode = pFuncNode; + + tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName)); - strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, - tListLen(pExp->pExpr->_function.functionName)); #if 1 // todo refactor: add the parameter for tbname function - if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { + const char* name = "tbname"; + int32_t len = strlen(name); + + if (!pFuncNode->pParameterList && (memcmp(pExprNode->_function.functionName, name, len) == 0) && + pExprNode->_function.functionName[len] == 0) { + pFuncNode->pParameterList = nodesMakeList(); ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); @@ -1250,6 +1257,17 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pOpNode->node.aliasName); pExp->pExpr->_optrRoot.pRootNode = pNode; + } else if (type == QUERY_NODE_CASE_WHEN) { + pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; + SCaseWhenNode* pCaseNode = (SCaseWhenNode*)pNode; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pCaseNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, + pType->precision, pCaseNode->node.aliasName); + pExp->pExpr->_optrRoot.pRootNode = pNode; } else { ASSERT(0); } @@ -1348,7 +1366,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); } else { char* udfName = pExpr->pExpr->_function.pFunctNode->functionName; - strncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN); fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet); } pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e527f7e7c1720a90dd34d1481fbda90c4d6be8a0..41f861cd38849c0e5c82f3bed9e5faff9e5813f9 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -616,7 +616,7 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { int32_t nOptrWithVal = 0; int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); - if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { + if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) { taosMemoryFreeClear(*pOutput); *len = 0; } @@ -701,10 +701,10 @@ int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { return 0; } -int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo mtInfo) { +int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) { memset(pCond, 0, sizeof(SQueryTableDataCond)); pCond->order = TSDB_ORDER_ASC; - pCond->numOfCols = mtInfo.schema->nCols; + pCond->numOfCols = pMtInfo->schema->nCols; pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); if (pCond->colList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -712,15 +712,15 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s } pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - pCond->suid = mtInfo.suid; + pCond->suid = pMtInfo->suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = sContext->snapVersion; for (int32_t i = 0; i < pCond->numOfCols; ++i) { - pCond->colList[i].type = mtInfo.schema->pSchema[i].type; - pCond->colList[i].bytes = mtInfo.schema->pSchema[i].bytes; - pCond->colList[i].colId = mtInfo.schema->pSchema[i].colId; + pCond->colList[i].type = pMtInfo->schema->pSchema[i].type; + pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes; + pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId; } return TSDB_CODE_SUCCESS; @@ -844,7 +844,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT taosArrayDestroy(pTaskInfo->tableqinfoList.pTableList); if (mtInfo.uid == 0) return 0; // no data - initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, mtInfo); + initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0}); @@ -856,15 +856,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); pTaskInfo->streamInfo.schema = mtInfo.schema; - qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %ld", mtInfo.uid, pOffset->ts); + qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %"PRId64, mtInfo.uid, pOffset->ts); } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; if (setForSnapShot(sContext, pOffset->uid) != 0) { - qError("setForSnapShot error. uid:%" PRIi64 " ,version:%" PRIi64, pOffset->uid); + qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); return -1; } - qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %ld", pOffset->uid); + qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %"PRId64, pOffset->uid, pOffset->ts); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; tsdbReaderClose(pInfo->dataReader); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8c1104f51988930776d8745fceb7f727289751eb..aaef411d2dfef41d6e1dc2c62280cafe18726d9d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -895,33 +895,6 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } #endif -static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { -#if 0 - SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; - uint32_t status = BLK_DATA_NOT_LOAD; - - int32_t numOfOutput = 0; // pTableScanInfo->numOfOutput; - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functionId = pCtx[i].functionId; - int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId; - - // group by + first/last should not apply the first/last block filter - if (functionId < 0) { - status |= BLK_DATA_DATA_LOAD; - return status; - } else { - // status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); - // if ((status & BLK_DATA_DATA_LOAD) == BLK_DATA_DATA_LOAD) { - // return status; - // } - } - } - - return status; -#endif - return 0; -} - int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NOT_LOAD; @@ -1113,15 +1086,24 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, int32_t status); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo) { if (pFilterNode == NULL || pBlock->info.rows == 0) { return; } - SFilterInfo* filter = NULL; + SFilterInfo* filter = pFilterInfo; + int64_t st = taosGetTimestampUs(); + +// pError("start filter"); // todo move to the initialization function - int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + int32_t code = 0; + bool needFree = false; + if (filter == NULL) { + needFree = true; + code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); + } + SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); @@ -1130,7 +1112,10 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM // todo the keep seems never to be True?? bool keep = filterExecute(filter, pBlock, &p, NULL, param1.numOfCols, &status); - filterFreeInfo(filter); + + if (needFree) { + filterFreeInfo(filter); + } extractQualifiedTupleByFilterResult(pBlock, p, keep, status); @@ -1790,7 +1775,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); + qDebug("%s fetch rsp received, index:%d, code:%s", pSourceDataInfo->taskId, index, tstrerror(code)); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -1845,6 +1830,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); if (NULL == pMsg) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); return pTaskInfo->code; } @@ -2479,7 +2465,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - doFilter(pAggInfo->pCondition, pInfo->pRes, NULL); + doFilter(pAggInfo->pCondition, pInfo->pRes, NULL, NULL); if (!hasRemainResults(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -2803,6 +2789,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId); blockDataCleanup(pInfo->pRes); + blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); + blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.groupId) { @@ -2873,7 +2861,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { break; } - doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo); + doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo, NULL); if (fillResult->info.rows > 0) { break; } @@ -3049,6 +3037,12 @@ void cleanupExprSupp(SExprSupp* pSupp) { destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs); taosMemoryFreeClear(pSupp->pExprInfo); } + + if (pSupp->pFilterInfo != NULL) { + filterFreeInfo(pSupp->pFilterInfo); + pSupp->pFilterInfo = NULL; + } + taosMemoryFree(pSupp->rowEntryInfoOffset); } @@ -3378,8 +3372,8 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; - pSchema->type = pColNode->node.resType.bytes; - strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + pSchema->bytes = pColNode->node.resType.bytes; + tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } // this the tags and pseudo function columns, we only keep the tag columns @@ -3393,7 +3387,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; - pSchema->type = pColNode->node.resType.bytes; + pSchema->bytes = pColNode->node.resType.bytes; strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index f3ae652987eb1ab133dbe7a211c20a7e36f3b48d..e62f72c8f8f1394adf3ae9a50816e4ba4057bc95 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -311,7 +311,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes, NULL); + doFilter(pInfo->pCondition, pRes, NULL, NULL); if (!hasRemainResults(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -991,6 +991,8 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFree(pInfo->partitionSup.keyBuf); cleanupExprSupp(&pInfo->scalarSup); + cleanupExprSupp(&pInfo->tbnameCalSup); + cleanupExprSupp(&pInfo->tagCalSup); blockDataDestroy(pInfo->pDelRes); taosMemoryFreeClear(param); } @@ -1037,6 +1039,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + if (pPartNode->pTags != NULL) { + int32_t numOfTags; + SExprInfo* pTagExpr = createExprInfo(pPartNode->pTags, NULL, &numOfTags); + if (pTagExpr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + } + int32_t keyLen = 0; code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index b4aa6de0a890c2c226eb0b6d74f3648b87dc7871..53cfa6c27a880230aa919b245a8a16687498000c 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -387,7 +387,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { break; } if (pJoinInfo->pCondAfterMerge != NULL) { - doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL); + doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL, NULL); } if (pRes->info.rows >= pOperator->resultInfo.threshold) { break; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index e9e6fed66aad43daf71a99613fb966d16461da76..fe6ee4129a9a0d0bd6dec2eb3e2fddc5452880da 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -315,7 +315,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } // do apply filter - doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL); + doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL, NULL); // when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint. if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) { @@ -325,7 +325,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } else { // do apply filter if (pRes->info.rows > 0) { - doFilter(pProjectInfo->pFilterNode, pRes, NULL); + doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL); if (pRes->info.rows == 0) { continue; } @@ -518,7 +518,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { } } - doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL); + doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL, NULL); size_t rows = pInfo->pRes->info.rows; if (rows > 0 || pOperator->status == OP_EXEC_DONE) { break; @@ -620,7 +620,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { } pRes->info.rows = 1; - doFilter(pProjectInfo->pFilterNode, pRes, NULL); + doFilter(pProjectInfo->pFilterNode, pRes, NULL, NULL); /*int32_t status = */ doIngroupLimitOffset(&pProjectInfo->limitInfo, 0, pRes, pOperator); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c20bb9581164de0a76fee5a1aec983c3802be203..15a7abf1ffa675fe46fb1e1f0c64e02fe7437185 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -284,6 +284,19 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, return true; } + +static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + if (pTableScanInfo->pseudoSup.numOfExprs > 0) { + SExprSupp* pSup = &pTableScanInfo->pseudoSup; + + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, + GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + } +} + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -312,8 +325,9 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->skipBlocks += 1; + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; @@ -322,6 +336,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -371,21 +386,11 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true); - - // currently only the tbname pseudo column - if (pTableScanInfo->pseudoSup.numOfExprs > 0) { - SExprSupp* pSup = &pTableScanInfo->pseudoSup; - - int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, - GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } - } + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampUs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, pOperator->exprSupp.pFilterInfo); double el = (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->readRecorder.filterTime += el; @@ -754,6 +759,11 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; + + if (pInfo->pFilterNode != NULL) { + code = filterInitFromNode((SNode*)pInfo->pFilterNode, &pOperator->exprSupp.pFilterInfo, 0); + } + pInfo->scanFlag = MAIN_SCAN; pInfo->pColMatchInfo = pColList; pInfo->currentGroupId = -1; @@ -1074,12 +1084,13 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow endWin = win; STimeWindow preWin = win; uint64_t groupId = gpIdCol[*pRowIndex]; + while (1) { if (hasGroup) { (*pRowIndex) += 1; @@ -1123,7 +1134,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 return NULL; } - doFilter(pInfo->pCondition, pResult, NULL); + doFilter(pInfo->pCondition, pResult, NULL, NULL); if (pResult->info.rows == 0) { continue; } @@ -1143,6 +1154,9 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 pResult->info.rows++; } } + + blockDataDestroy(tmpBlock); + if (pResult->info.rows > 0) { pResult->info.calWin = pInfo->updateWin; return pResult; @@ -1315,6 +1329,11 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* ASSERT(pResBlock->info.rows == 1); // build tagArray + /*SArray* tagArray = taosArrayInit(0, sizeof(void*));*/ + /*STagVal tagVal = {*/ + /*.cid = 0,*/ + /*.type = 0,*/ + /*};*/ // build STag // set STag @@ -1469,7 +1488,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } if (filter) { - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); } blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataFreeRes((SSDataBlock*)pBlock); @@ -1533,7 +1552,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { tsdbReaderClose(pTSInfo->dataReader); pTSInfo->dataReader = NULL; tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); - qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1); + qDebug("queue scan tsdb over, switch to wal ver %"PRId64, pTaskInfo->streamInfo.snapshotVer + 1); if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) { return NULL; } @@ -1891,7 +1910,7 @@ FETCH_NEXT_BLOCK: } } - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { @@ -2164,6 +2183,19 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } + if (pTableScanNode->pTags != NULL) { + int32_t numOfTags; + SExprInfo* pTagExpr = createExprInfo(pTableScanNode->pTags, NULL, &numOfTags); + if (pTagExpr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + } + pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2361,7 +2393,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } @@ -2542,16 +2574,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SMetaReader smrChildTable = {0}; metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - metaGetTableEntryByName(&smrChildTable, condTableName); + int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + return NULL; + } + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { metaReaderClear(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } + SMetaReader smrSuperTable = {0}; metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByUid + return NULL; + } + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); metaReaderClear(&smrSuperTable); metaReaderClear(&smrChildTable); @@ -2958,7 +3001,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { while (1) { int64_t startTs = taosGetTimestampUs(); - strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); strcpy(pInfo->req.user, pInfo->pUser); int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); @@ -3434,7 +3477,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, NULL); double el = (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->readRecorder.filterTime += el; @@ -3769,6 +3812,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pColList); goto _error; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index ed3247e10ed773c9de2c996ac8eb366d0c9b36bf..0d5f52103420dfe874d6e20514e1891db55a2362 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -216,7 +216,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { return NULL; } - doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo); + doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo, NULL); if (blockDataGetNumOfRows(pBlock) == 0) { continue; } @@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataDestroy(p); - qDebug("%s get sorted block, groupId:%0x" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, + qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 5be991818529dc8158881ee82084fdb7c9565954..8b716531e57455a240c9e0054aae06b732320661 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1501,7 +1501,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { } doStreamFillImpl(pOperator); - doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo); + doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo, NULL); pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; if (pInfo->pRes->info.rows > 0) { break; @@ -1681,7 +1681,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); pInfo->pCondition = pPhyFillNode->node.pConditions; pInfo->pColMatchColInfo = pColMatchColInfo; - initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pInfo->srcRowIndex = 0; pOperator->name = "FillOperator"; @@ -1693,7 +1697,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo, NULL, NULL, NULL); - int32_t code = appendDownstream(pOperator, &downstream, 1); + code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1702,5 +1706,6 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi _error: destroyStreamFillOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; return NULL; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index df16914cb87c0bbd7d25f6e01985160be25bd74d..cbbd874bb911c2df82f54cc6eecc7e2956d6e362 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1277,7 +1277,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -1315,7 +1315,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBlock, NULL); + doFilter(pInfo->pCondition, pBlock, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -1420,8 +1420,9 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* pGpDatas = (uint64_t*)pGpCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; + STimeWindow win = {0}; if (IS_FINAL_OP(pInfo)) { win.skey = startTsCols[i]; @@ -2019,7 +2020,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -2062,7 +2063,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes, NULL); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL, NULL); bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { @@ -2841,14 +2842,13 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); - pInfo->twAggSup.waterMark = pSessionNode->window.watermark; pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType; pInfo->gap = pSessionNode->gap; @@ -4665,7 +4665,6 @@ void destroyStreamStateOperatorInfo(void* param) { SStreamSessionAggOperatorInfo* pChInfo = pChild->info; destroyStreamSessionAggOperatorInfo(pChInfo); taosMemoryFreeClear(pChild); - taosMemoryFreeClear(pChInfo); } } colDataDestroy(&pInfo->twAggSup.timeWindowData); @@ -5251,7 +5250,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pSup->pCtx, pBlock, pIaInfo->inputOrder, scanFlag, true); doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes); - doFilter(pMiaInfo->pCondition, pRes, NULL); + doFilter(pMiaInfo->pCondition, pRes, NULL, NULL); if (pRes->info.rows >= pOperator->resultInfo.capacity) { break; } @@ -5828,27 +5827,30 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->isFinal = false; - if (pIntervalPhyNode->window.pExprs != NULL) { - int32_t numOfScalar = 0; - SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } + pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); SExprSupp* pSup = &pOperator->exprSupp; + + initBasicInfo(&pInfo->binfo, pResBlock); + initStreamFunciton(pSup->pCtx, pSup->numOfExprs); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); - initStreamFunciton(pSup->pCtx, pSup->numOfExprs); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + if (pIntervalPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } pInfo->invertible = allInvertible(pSup->pCtx, numOfCols); pInfo->invertible = false; diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 16fd11f97d330fa3bc2622e2f7671fa532dce61f..134862d88cc3cda6988aa3b601d24bd813ffadc2 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -18,7 +18,7 @@ #include "tlog.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 -#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 16L) #define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) #define GET_SHASH_NODE_KEY(_n, _dl) ((char *)(_n) + sizeof(SHNode) + (_dl)) @@ -104,7 +104,7 @@ static void tSimpleHashTableResize(SSHashObj *pHashObj) { int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { - uDebug("current capacity:%zu, maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", + uDebug("current capacity:%"PRIzu", maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", pHashObj->capacity, HASH_MAX_CAPACITY); return; } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index dac9b857404d202ec56e9d2e7b57c954d1a3c143..b9b365fb42c62ec9400b85450c3bca40ac920730 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -131,7 +131,7 @@ static int32_t valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { COPY_SCALAR_FIELD(placeholderNo); COPY_SCALAR_FIELD(typeData); COPY_SCALAR_FIELD(unit); - if (!pSrc->translate) { + if (!pSrc->translate || pSrc->isNull) { return TSDB_CODE_SUCCESS; } switch (pSrc->node.resType.type) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index e401a3da7fe4fdb9815eae3fedfec9e5b281689d..f0f1fc99756bacf892e9a7ce48677b94a06c0fd3 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3011,7 +3011,7 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkValueIsNull, pNode->isNull); } - if (TSDB_CODE_SUCCESS == code && pNode->translate) { + if (TSDB_CODE_SUCCESS == code && pNode->translate && !pNode->isNull) { code = datumToJson(pNode, pJson); } @@ -3161,7 +3161,7 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkValueIsNull, &pNode->isNull); } - if (TSDB_CODE_SUCCESS == code && pNode->translate) { + if (TSDB_CODE_SUCCESS == code && pNode->translate && !pNode->isNull) { code = jsonToDatum(pJson, pNode); } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index cdc4e66e4247615697a6090e2ad1c1e8a1659529..9e89955ae5394db0923ab8eb82d99bde755a8e53 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -478,7 +478,10 @@ static int32_t tlvDecodeValueEnum(STlvDecoder* pDecoder, void* pValue, int16_t l return code; } -static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue) { +static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue, int32_t size) { + if (pTlv->len > size - 1) { + return TSDB_CODE_FAILED; + } memcpy(pValue, pTlv->value, pTlv->len); return TSDB_CODE_SUCCESS; } @@ -919,9 +922,14 @@ static int32_t msgToDatum(STlv* pTlv, void* pObj) { } break; } - case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_JSON: { + if (pTlv->len <= 0 || pTlv->len > TSDB_MAX_JSON_TAG_LEN) { + code = TSDB_CODE_FAILED; + break; + } code = tlvDecodeDynBinary(pTlv, (void**)&pNode->datum.p); break; + } case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: // todo @@ -1097,7 +1105,7 @@ static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); break; case FUNCTION_CODE_FUNCTION_NAME: - code = tlvDecodeCStr(pTlv, pNode->functionName); + code = tlvDecodeCStr(pTlv, pNode->functionName, sizeof(pNode->functionName)); break; case FUNCTION_CODE_FUNCTION_ID: code = tlvDecodeI32(pTlv, &pNode->funcId); @@ -1226,10 +1234,10 @@ static int32_t msgToName(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI32(pTlv, &pNode->acctId); break; case NAME_CODE_DB_NAME: - code = tlvDecodeCStr(pTlv, pNode->dbname); + code = tlvDecodeCStr(pTlv, pNode->dbname, sizeof(pNode->dbname)); break; case NAME_CODE_TABLE_NAME: - code = tlvDecodeCStr(pTlv, pNode->tname); + code = tlvDecodeCStr(pTlv, pNode->tname, sizeof(pNode->tname)); break; default: break; @@ -1538,7 +1546,7 @@ static int32_t msgToEp(STlvDecoder* pDecoder, void* pObj) { tlvForEach(pDecoder, pTlv, code) { switch (pTlv->type) { case EP_CODE_FQDN: - code = tlvDecodeCStr(pTlv, pNode->fqdn); + code = tlvDecodeCStr(pTlv, pNode->fqdn, sizeof(pNode->fqdn)); break; case EP_CODE_port: code = tlvDecodeU16(pTlv, &pNode->port); @@ -3207,7 +3215,7 @@ static int32_t msgToPhysiQueryInsertNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI8(pTlv, &pNode->tableType); break; case PHY_QUERY_INSERT_CODE_TABLE_NAME: - code = tlvDecodeCStr(pTlv, pNode->tableName); + code = tlvDecodeCStr(pTlv, pNode->tableName, sizeof(pNode->tableName)); break; case PHY_QUERY_INSERT_CODE_VG_ID: code = tlvDecodeI32(pTlv, &pNode->vgId); @@ -3284,10 +3292,10 @@ static int32_t msgToPhysiDeleteNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeI8(pTlv, &pNode->tableType); break; case PHY_DELETER_CODE_TABLE_FNAME: - code = tlvDecodeCStr(pTlv, pNode->tableFName); + code = tlvDecodeCStr(pTlv, pNode->tableFName, sizeof(pNode->tableFName)); break; case PHY_DELETER_CODE_TS_COL_NAME: - code = tlvDecodeCStr(pTlv, pNode->tsColName); + code = tlvDecodeCStr(pTlv, pNode->tsColName, sizeof(pNode->tsColName)); break; case PHY_DELETER_CODE_DELETE_TIME_RANGE: code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->deleteTimeRange); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 6e964fb53a7e36b435169f04daced63d43f6a786..bc0c4d42bbccc66e8a00179d523f86159f03e33e 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -209,6 +209,10 @@ int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId) { } SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId); + if (NULL == pAllocator) { + nodesError("allocator id %" PRIx64 " weak reference failed", allocatorId); + return -1; + } return pAllocator->self; } @@ -1604,7 +1608,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || - QUERY_NODE_FUNCTION == type || QUERY_NODE_LOGIC_CONDITION == type); + QUERY_NODE_FUNCTION == type || QUERY_NODE_LOGIC_CONDITION == type || QUERY_NODE_CASE_WHEN == type); } bool nodesIsUnaryOp(const SOperatorNode* pOp) { @@ -1716,9 +1720,10 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; int32_t len = 0; if ('\0' == pCol->tableAlias[0]) { - len = sprintf(name, "%s", pCol->colName); + len = snprintf(name, sizeof(name), "%s", pCol->colName); + } else { + len = snprintf(name, sizeof(name), "%s.%s", pCol->tableAlias, pCol->colName); } - len = sprintf(name, "%s.%s", pCol->tableAlias, pCol->colName); if (NULL == taosHashGet(pCxt->pColHash, name, len)) { pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, NULL, 0); if (TSDB_CODE_SUCCESS == pCxt->errCode) { diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h index 787abf287e3bcf4f42e505d40bea0b7230cfc7e7..fb4b46aa35094fd3ac65171c5499c038eb92d233 100644 --- a/source/libs/parser/inc/parToken.h +++ b/source/libs/parser/inc/parToken.h @@ -175,8 +175,6 @@ _end: void taosCleanupKeywordsTable(); -SToken tscReplaceStrToken(char **str, SToken *token, const char *newToken); - SToken taosTokenDup(SToken *pToken, char *buf, int32_t len); #ifdef __cplusplus diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 2e6f83936848b044b4f13f894ec44672db9bfec1..15b939f14b3bfb32298fac514d22655a8b00d9fe 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -518,7 +518,7 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTa if (NULL != pDbName) { COPY_STRING_FORM_ID_TOKEN(realTable->table.dbName, pDbName); } else { - strcpy(realTable->table.dbName, pCxt->pQueryCxt->db); + snprintf(realTable->table.dbName, sizeof(realTable->table.dbName), "%s", pCxt->pQueryCxt->db); } if (NULL != pTableAlias && TK_NK_NIL != pTableAlias->type) { COPY_STRING_FORM_ID_TOKEN(realTable->table.tableAlias, pTableAlias); @@ -599,7 +599,7 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { state->pCol = createPrimaryKeyCol(pCxt, NULL); if (NULL == state->pCol) { nodesDestroyNode((SNode*)state); - CHECK_OUT_OF_MEM(state->pCol); + CHECK_OUT_OF_MEM(NULL); } state->pExpr = pExpr; return (SNode*)state; @@ -613,7 +613,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode interval->pCol = createPrimaryKeyCol(pCxt, NULL); if (NULL == interval->pCol) { nodesDestroyNode((SNode*)interval); - CHECK_OUT_OF_MEM(interval->pCol); + CHECK_OUT_OF_MEM(NULL); } interval->pInterval = pInterval; interval->pOffset = pOffset; @@ -631,7 +631,7 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { fill->pWStartTs = nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == fill->pWStartTs) { nodesDestroyNode((SNode*)fill); - CHECK_OUT_OF_MEM(fill->pWStartTs); + CHECK_OUT_OF_MEM(NULL); } strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstart"); return (SNode*)fill; @@ -1495,10 +1495,10 @@ SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool igno CHECK_OUT_OF_MEM(pStmt); pStmt->indexType = type; pStmt->ignoreExists = ignoreExists; - strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName); - strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName); - strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName); + snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName); + snprintf(pStmt->dbName, sizeof(pStmt->dbName), "%s", ((SRealTableNode*)pRealTable)->table.dbName); + snprintf(pStmt->tableName, sizeof(pStmt->tableName), "%s", ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pIndexName); nodesDestroyNode(pRealTable); pStmt->pCols = pCols; @@ -1524,8 +1524,8 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* SDropIndexStmt* pStmt = (SDropIndexStmt*)nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->ignoreNotExists = ignoreNotExists; - strcpy(pStmt->indexDbName, ((SRealTableNode*)pIndexName)->table.dbName); - strcpy(pStmt->indexName, ((SRealTableNode*)pIndexName)->table.tableName); + snprintf(pStmt->indexDbName, sizeof(pStmt->indexDbName), "%s", ((SRealTableNode*)pIndexName)->table.dbName); + snprintf(pStmt->indexName, sizeof(pStmt->indexName), "%s", ((SRealTableNode*)pIndexName)->table.tableName); nodesDestroyNode(pIndexName); return (SNode*)pStmt; } @@ -1818,7 +1818,7 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb SNode* createFuncForDelete(SAstCreateContext* pCxt, const char* pFuncName) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(pFunc); - strcpy(pFunc->functionName, pFuncName); + snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName); if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt, NULL))) { nodesDestroyNode((SNode*)pFunc); CHECK_OUT_OF_MEM(NULL); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index eec20f91e4ab75c645fd0584ca7213fbd5356514..5b861ef79acbe0bc59d544739297e98e2be7bcc6 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -378,7 +378,9 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time } else if (pToken->type == TK_TODAY) { ts = taosGetTimestampToday(timePrec); } else if (pToken->type == TK_NK_INTEGER) { - toInteger(pToken->z, pToken->n, 10, &ts); + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } } else { // parse the RFC-3339/ISO-8601 timestamp format string if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); @@ -591,8 +593,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_BIGINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); } return func(pMsgBuf, &iv, pSchema->bytes, param); } @@ -894,10 +894,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_BIGINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); } - val->i64 = iv; break; } @@ -1173,7 +1170,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName* NEXT_TOKEN(pCxt->pSql, sToken); SName sname; - createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); + CHECK_CODE(createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(&sname, dbFName); strcpy(pCxt->sTableName, sname.tname); @@ -1605,7 +1602,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (!pCxt->pComCxt->needMultiParse) { continue; } else { - parserInfo("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); + parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); break; } } @@ -1613,7 +1610,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); } - parserInfo("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); @@ -1650,7 +1647,7 @@ static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) { pCxt->pComCxt->needMultiParse = false; return TSDB_CODE_SUCCESS; } - parserInfo("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); // merge according to vgId if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index e8c877bed2d69346cd718625688b0db3de85ae69..997e680c99c8a93fad53543cd9b5aeec2c9e1154 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -// clang-format off + #include "parInsertData.h" #include "catalog.h" @@ -25,8 +25,8 @@ (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; + TSKEY skey; + void* payloadAddr; int16_t index; } SBlockKeyTuple; @@ -194,8 +194,8 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { SEncoder coder = {0}; - char* pBuf; - int32_t len; + char* pBuf; + int32_t len; int32_t ret = 0; tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret); @@ -211,19 +211,19 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) } } - pBuf= pBlocks->pData + pBlocks->size; + pBuf = pBlocks->pData + pBlocks->size; tEncoderInit(&coder, pBuf, len); - tEncodeSVCreateTbReq(&coder, pCreateTbReq); + int32_t code = tEncodeSVCreateTbReq(&coder, pCreateTbReq); tEncoderClear(&coder); - pBlocks->size += len; pBlocks->createTbReqLen = len; - return TSDB_CODE_SUCCESS; + + return code; } -int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, int32_t rowSize, - STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, +int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, + int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen); @@ -272,12 +272,12 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { } taosMemoryFreeClear(pDataBlock->pData); -// if (!pDataBlock->cloned) { - // free the refcount for metermeta - taosMemoryFreeClear(pDataBlock->pTableMeta); + // if (!pDataBlock->cloned) { + // free the refcount for metermeta + taosMemoryFreeClear(pDataBlock->pTableMeta); - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); -// } + destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + // } taosMemoryFreeClear(pDataBlock); } @@ -687,16 +687,16 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p STableDataBlocks** p = taosHashIterate(pHashObj, NULL); STableDataBlocks* pOneTableBlock = *p; SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - SBlockRowMerger *pBlkRowMerger = NULL; + SBlockRowMerger* pBlkRowMerger = NULL; while (pOneTableBlock) { SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; - pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId - int32_t ret = - getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, - pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); + pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId + int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), + TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, + pVnodeDataBlockList, NULL); if (ret != TSDB_CODE_SUCCESS) { tdFreeSBlockRowMerger(pBlkRowMerger); taosHashCleanup(pVnodeDataBlockHashList); @@ -708,7 +708,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p // the maximum expanded size in byte when a row-wise data is converted to SDataRow format int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + pOneTableBlock->createTbReqLen; + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + + pOneTableBlock->createTbReqLen; if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); @@ -861,7 +862,7 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst); if (pBlock->pTableMeta) { - void *pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); + void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); if (NULL == pNewMeta) { taosMemoryFreeClear(*pDst); return TSDB_CODE_OUT_OF_MEMORY; @@ -887,20 +888,18 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgI } pBlock->vgId = vgId; - + if (pBlock->pTableMeta) { pBlock->pTableMeta->uid = uid; pBlock->pTableMeta->vgId = vgId; } - + memset(pBlock->pData, 0, sizeof(SSubmitBlk)); return TSDB_CODE_SUCCESS; } -STableMeta *qGetTableMetaInDataBlock(void* pDataBlock) { - return ((STableDataBlocks*)pDataBlock)->pTableMeta; -} +STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; } void qFreeStmtDataBlock(void* pDataBlock) { if (pDataBlock == NULL) { diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 345b9cc5b9e29449d009ab9c0911b1438e44b8b3..76c01cfd90a9a3b52c25d537ee65708295191c15 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -613,27 +613,6 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { return 0; } -SToken tscReplaceStrToken(char** str, SToken* token, const char* newToken) { - char* src = *str; - size_t nsize = strlen(newToken); - int32_t size = (int32_t)strlen(*str) - token->n + (int32_t)nsize + 1; - int32_t bsize = (int32_t)((uint64_t)token->z - (uint64_t)src); - SToken ntoken; - - *str = taosMemoryCalloc(1, size); - - strncpy(*str, src, bsize); - strcat(*str, newToken); - strcat(*str, token->z + token->n); - - ntoken.n = (uint32_t)nsize; - ntoken.z = *str + bsize; - - taosMemoryFreeClear(src); - - return ntoken; -} - SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) { SToken t0 = {0}; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b398b645b393bbf34bce0515acc524b27fc3b66f..4739a852c0480fc9c1e4242590674a96d7eab627 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -292,9 +292,13 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { } else { do { SArray* pTables = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + if (NULL == pTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } if (pCxt->currLevel == currTotalLevel) { taosArrayPush(pTables, &pTable); if (hasSameTableAlias(pTables)) { + taosArrayDestroy(pTables); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS, "Not unique table/alias: '%s'", ((STableNode*)pTable)->tableAlias); } @@ -308,7 +312,7 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) { SFullDatabaseName name = {0}; - strcpy(name.fullDbName, pFullDbName); + snprintf(name.fullDbName, sizeof(name.fullDbName), "%s", pFullDbName); return taosHashPut(pDbs, pFullDbName, strlen(pFullDbName), &name, sizeof(SFullDatabaseName)); } @@ -540,15 +544,17 @@ static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray if (TSDB_CODE_SUCCESS == code) { code = collectUseTable(pName, pCxt->pTables); } - if (pParCxt->async) { - code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); - } else { - SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter, - .requestId = pParCxt->requestId, - .requestObjRefId = pParCxt->requestRid, - .mgmtEps = pParCxt->mgmtEpSet}; + if (TSDB_CODE_SUCCESS == code) { + if (pParCxt->async) { + code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); + } else { + SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter, + .requestId = pParCxt->requestId, + .requestObjRefId = pParCxt->requestRid, + .mgmtEps = pParCxt->mgmtEpSet}; - code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes); + code = catalogGetTableIndex(pParCxt->pCatalog, &conn, pName, pIndexes); + } } if (TSDB_CODE_SUCCESS != code) { parserError("0x%" PRIx64 " getTableIndex error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId, @@ -990,9 +996,9 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal) return pCxt->errCode; } if (IS_UNSIGNED_NUMERIC_TYPE(pVal->node.resType.type)) { - pVal->datum.i = pVal->datum.u; + pVal->datum.i = (int64_t)pVal->datum.u; } else if (IS_FLOAT_TYPE(pVal->node.resType.type)) { - pVal->datum.i = pVal->datum.d; + pVal->datum.i = (int64_t)pVal->datum.d; } else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) { pVal->datum.i = pVal->datum.b; } @@ -1069,7 +1075,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_BIGINT: { code = toInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.i); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_BIGINT(pVal->datum.i))) { + if (strict && TSDB_CODE_SUCCESS != code) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(int64_t*)&pVal->typeData = pVal->datum.i; @@ -1888,6 +1894,7 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW pWhenThen->pWhen = pIsTrue; } if (first) { + first = false; pCaseWhen->node.resType = ((SExprNode*)pNode)->resType; } else if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { SNode* pCastFunc = NULL; @@ -2350,7 +2357,8 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo } static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSelect) { - if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable)) { + if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) || + TSDB_SYSTEM_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType) { return TSDB_CODE_SUCCESS; } @@ -3429,10 +3437,10 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { return NULL; } pCol->node.resType = ((SExprNode*)pNode)->resType; - strcpy(pCol->tableAlias, pTableAlias); - strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName); - strcpy(pCol->node.aliasName, pCol->colName); - strcpy(pCol->node.userAlias, ((SExprNode*)pNode)->userAlias); + snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pTableAlias); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", ((SExprNode*)pNode)->aliasName); + snprintf(pCol->node.aliasName, sizeof(pCol->node.aliasName), "%s", pCol->colName); + snprintf(pCol->node.userAlias, sizeof(pCol->node.userAlias), "%s", ((SExprNode*)pNode)->userAlias); return (SNode*)pCol; } @@ -3739,8 +3747,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int32_t val, int32_t minVal, int32_t maxVal) { if (val >= 0 && (val < minVal || val > maxVal)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %" PRId64 " valid range: [%d, %d]", pName, - val, minVal, maxVal); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %d valid range: [%d, %d]", pName, val, + minVal, maxVal); } return TSDB_CODE_SUCCESS; } @@ -3877,7 +3885,7 @@ static int32_t checkDbStrictOption(STranslateContext* pCxt, SDatabaseOptions* pO static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t v1, int32_t v2) { if (val >= 0 && val != v1 && val != v2) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option %s: %" PRId64 ", only %d, %d allowed", pName, val, v1, v2); + "Invalid option %s: %d, only %d, %d allowed", pName, val, v1, v2); } return TSDB_CODE_SUCCESS; } @@ -4462,8 +4470,8 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(pTable->table.dbName, pInfo->pDbName); - strcpy(pTable->table.tableName, pInfo->pTableName); + snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); + snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); pSelect->pFromTable = (SNode*)pTable; @@ -4939,9 +4947,9 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt alterReq.superUser = 0; alterReq.enable = pStmt->enable; alterReq.sysInfo = pStmt->sysinfo; - strcpy(alterReq.pass, pStmt->password); + snprintf(alterReq.pass, sizeof(alterReq.pass), "%s", pStmt->password); if (NULL != pCxt->pParseCxt->db) { - strcpy(alterReq.dbname, pCxt->pParseCxt->db); + snprintf(alterReq.dbname, sizeof(alterReq.dbname), "%s", pCxt->pParseCxt->db); } return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &alterReq); @@ -6048,7 +6056,7 @@ static SNode* createProjectCol(const char* pProjCol) { if (NULL == pCol) { return NULL; } - strcpy(pCol->colName, pProjCol); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", pProjCol); return (SNode*)pCol; } @@ -6081,9 +6089,9 @@ static int32_t createSimpleSelectStmt(const char* pDb, const char* pTable, int32 nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(pRealTable->table.dbName, pDb); - strcpy(pRealTable->table.tableName, pTable); - strcpy(pRealTable->table.tableAlias, pTable); + snprintf(pRealTable->table.dbName, sizeof(pRealTable->table.dbName), "%s", pDb); + snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable); + snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable); pSelect->pFromTable = (SNode*)pRealTable; if (numOfProjs >= 0) { @@ -7124,8 +7132,9 @@ static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* p pReq->newComment = strdup(pStmt->pOptions->comment); if (NULL == pReq->newComment) { code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pReq->newCommentLen = strlen(pReq->newComment); } - pReq->newCommentLen = strlen(pReq->newComment); } else { pReq->newCommentLen = -1; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index dab1f015748187ca1d3870ad6ea821acbbbc01e8..ee82e6c04beb62e58151d23725c1a4bc217e827b 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -381,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); + taosMemoryFree(tmp); goto end; } val.nData = valLen; @@ -652,8 +653,8 @@ static int32_t buildCatalogReqForInsert(SParseContext* pCxt, const SParseMetaCac } SUserAuthInfo auth = {0}; - strcpy(auth.user, pCxt->pUser); - strcpy(auth.dbFName, p->dbFName); + snprintf(auth.user, sizeof(auth.user), "%s", pCxt->pUser); + snprintf(auth.dbFName, sizeof(auth.dbFName), "%s", p->dbFName); auth.type = AUTH_TYPE_WRITE; taosArrayPush(pCatalogReq->pUser, &auth); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index e6868f0cebfb5de7bf7321822c5928a6f67cdac7..9e1a01b13ddc1313bff458ed43c83b0d9e005d83 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -66,7 +66,8 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { switch (nodeType(*pNode)) { case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: - case QUERY_NODE_FUNCTION: { + case QUERY_NODE_FUNCTION: + case QUERY_NODE_CASE_WHEN: { SRewriteExprCxt* pCxt = (SRewriteExprCxt*)pContext; SNode* pExpr; int32_t index = 0; @@ -118,6 +119,17 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } +static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { + nodesWalkExpr(pExpr, doNameExpr, NULL); + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL}; + cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); + if (TSDB_CODE_SUCCESS == cxt.errCode) { + nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); + nodesClearList(cxt.pExprs); + } + return cxt.errCode; +} + static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; @@ -459,9 +471,9 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) return NULL; } pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", pExpr->aliasName); if (NULL != pStmtName) { - strcpy(pCol->tableAlias, pStmtName); + snprintf(pCol->tableAlias, sizeof(pCol->tableAlias), "%s", pStmtName); } return pCol; } @@ -711,8 +723,13 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo nodesDestroyNode((SNode*)pWindow); return TSDB_CODE_OUT_OF_MEMORY; } - - return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); + // rewrite the expression in subsequent clauses + int32_t code = rewriteExprForSelect(pWindow->pStateExpr, pSelect, SQL_CLAUSE_WINDOW); + if (TSDB_CODE_SUCCESS == code) { + code = createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); + } + + return code; } static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionWindowNode* pSession, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 50f1f4b3691d4cdf474a9ecad3ac445e0e0a6af9..7fd90dd4e8003e91fde947f01f8ef463a2086286 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -124,9 +124,8 @@ static void optSetParentOrder(SLogicNode* pNode, EOrder order) { EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - // *((bool*)pContext) = - // (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); - *((bool*)pContext) = true; + *((bool*)pContext) = + (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; @@ -1124,7 +1123,7 @@ static int32_t sortPriKeyOptGetSequencingNodes(SLogicNode* pNode, SNodeList** pS bool notOptimize = false; int32_t code = sortPriKeyOptGetSequencingNodesImpl(pNode, ¬Optimize, pSequencingNodes); if (TSDB_CODE_SUCCESS != code || notOptimize) { - nodesClearList(*pSequencingNodes); + NODES_CLEAR_LIST(*pSequencingNodes); } return code; } @@ -1361,74 +1360,6 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo return code; } -static SNode* smaIndexOptCreateWStartTs() { - SFunctionNode* pWStart = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pWStart) { - return NULL; - } - strcpy(pWStart->functionName, "_wstart"); - snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart); - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pWStart, NULL, 0)) { - nodesDestroyNode((SNode*)pWStart); - return NULL; - } - return (SNode*)pWStart; -} - -static int32_t smaIndexOptCreateMergeKey(SNode* pCol, SNodeList** pMergeKeys) { - SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); - if (NULL == pMergeKey) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pMergeKey->pExpr = nodesCloneNode(pCol); - if (NULL == pMergeKey->pExpr) { - nodesDestroyNode((SNode*)pMergeKey); - return TSDB_CODE_OUT_OF_MEMORY; - } - pMergeKey->order = ORDER_ASC; - pMergeKey->nullOrder = NULL_ORDER_FIRST; - return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey); -} - -static int32_t smaIndexOptRewriteInterval(SWindowLogicNode* pInterval, int32_t wstrartIndex, SNodeList** pMergeKeys) { - if (wstrartIndex < 0) { - SNode* pWStart = smaIndexOptCreateWStartTs(); - if (NULL == pWStart) { - return TSDB_CODE_OUT_OF_MEMORY; - } - int32_t code = createColumnByRewriteExpr(pWStart, &pInterval->node.pTargets); - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode(pWStart); - return code; - } - wstrartIndex = LIST_LENGTH(pInterval->node.pTargets) - 1; - } - return smaIndexOptCreateMergeKey(nodesListGetNode(pInterval->node.pTargets, wstrartIndex), pMergeKeys); -} - -static int32_t smaIndexOptApplyIndexExt(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { - SWindowLogicNode* pInterval = (SWindowLogicNode*)pScan->node.pParent; - SNodeList* pMergeTargets = nodesCloneList(pInterval->node.pTargets); - if (NULL == pMergeTargets) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SLogicNode* pSmaScan = NULL; - SLogicNode* pMerge = NULL; - SNodeList* pMergeKeys = NULL; - int32_t code = smaIndexOptRewriteInterval(pInterval, wstrartIndex, &pMergeKeys); - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); - } - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateMerge(pScan->node.pParent, pMergeKeys, pMergeTargets, &pMerge); - } - if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptRecombinationNode(pLogicSubplan, pScan->node.pParent, pMerge, pSmaScan); - } - return code; -} - static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pSmaCols, int32_t wstrartIndex) { SLogicNode* pSmaScan = NULL; @@ -1559,7 +1490,7 @@ static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) { return NULL; } - strcpy(pFunc->functionName, pFuncName); + snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName); if (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) { SColumnNode* pCol = (SColumnNode*)pNode; partTagsSetAlias(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), pCol->tableAlias, pCol->colName); @@ -2028,7 +1959,7 @@ static SNode* rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNode if (NULL != pSelectValue) { strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName); } else { - snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, pFunc); + snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, (void*)pFunc); } int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pCol)); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 2935c647aa571f7d29445c3f3cded18fae58eebb..8ff8538c0dbe934ba22beda84dce6c5fdc99749f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -67,7 +67,7 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const S if (NULL == pSlot) { return NULL; } - strcpy(pSlot->name, pName); + snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = reserve; @@ -663,13 +663,17 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc); } - SNodeList* condCols = nodesMakeList(); if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) { - code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &condCols); - } - if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockSlots(pCxt, condCols, pJoin->node.pOutputDataBlockDesc); - nodesDestroyList(condCols); + SNodeList* pCondCols = nodesMakeList(); + if (NULL == pCondCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &pCondCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pCondCols, pJoin->node.pOutputDataBlockDesc); + } + nodesDestroyList(pCondCols); } if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) { @@ -761,7 +765,8 @@ static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) { return collectAndRewrite(pCxt, pNode); } case QUERY_NODE_OPERATOR: - case QUERY_NODE_LOGIC_CONDITION: { + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_CASE_WHEN: { return collectAndRewrite(pCxt, pNode); } case QUERY_NODE_FUNCTION: { @@ -1633,7 +1638,7 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl pSubplan->subplanType = pLogicSubplan->subplanType; pSubplan->level = pLogicSubplan->level; if (NULL != pCxt->pPlanCxt->pUser) { - strcpy(pSubplan->user, pCxt->pPlanCxt->pUser); + snprintf(pSubplan->user, sizeof(pSubplan->user), "%s", pCxt->pPlanCxt->pUser); } return pSubplan; } @@ -1824,7 +1829,7 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNode* pSubplan, int32_t lev return TSDB_CODE_OUT_OF_MEMORY; } } - return nodesListStrictAppend(pGroup->pNodeList, (SNode*)pSubplan); + return nodesListAppend(pGroup->pNodeList, (SNode*)pSubplan); } static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pParent, diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 74ed3b57a4417aa43662bb722d3b78f3b13ee002..9567a2388f0695c9f2a89f433d921a4cbda35910 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -342,7 +342,7 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWStart->functionName, "_wstart"); - snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart); + snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, (void*)pWStart); int32_t code = fmGetFuncInfo(pWStart, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pFuncs, (SNode*)pWStart); @@ -367,7 +367,7 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWEnd->functionName, "_wend"); - snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, pWEnd); + snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, (void*)pWEnd); int32_t code = fmGetFuncInfo(pWEnd, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pWin->pFuncs, (SNode*)pWEnd); @@ -389,22 +389,19 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic SNode* pConditions = pMergeWindow->node.pConditions; pMergeWindow->node.pConditions = NULL; - int32_t code = TSDB_CODE_SUCCESS; SWindowLogicNode* pPartWin = (SWindowLogicNode*)nodesCloneNode((SNode*)pMergeWindow); if (NULL == pPartWin) { - code = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pPartWin->node.groupAction = GROUP_ACTION_KEEP; + pMergeWindow->node.pTargets = pTargets; + pMergeWindow->node.pConditions = pConditions; + pPartWin->node.pChildren = pChildren; + splSetParent((SLogicNode*)pPartWin); - if (TSDB_CODE_SUCCESS == code) { - pMergeWindow->node.pTargets = pTargets; - pMergeWindow->node.pConditions = pConditions; - pPartWin->node.pChildren = pChildren; - splSetParent((SLogicNode*)pPartWin); - code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs); - } int32_t index = 0; + int32_t code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { code = stbSplAppendWStart(pPartWin->pFuncs, &index); } @@ -721,15 +718,16 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO SNode* pConditions = pMergeAgg->node.pConditions; pMergeAgg->node.pConditions = NULL; - int32_t code = TSDB_CODE_SUCCESS; SAggLogicNode* pPartAgg = (SAggLogicNode*)nodesCloneNode((SNode*)pMergeAgg); if (NULL == pPartAgg) { - code = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pPartAgg->node.groupAction = GROUP_ACTION_KEEP; - if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL != pGroupKeys) { pPartAgg->pGroupKeys = pGroupKeys; code = createColumnByRewriteExprs(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets); } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index ac1589bec02a738ccb4e1f7d03b333628d012efe..a13e959a369ff03141e7c1a612529defdb1b5321 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -53,7 +53,8 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { } case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: - case QUERY_NODE_FUNCTION: { + case QUERY_NODE_FUNCTION: + case QUERY_NODE_CASE_WHEN: { SExprNode* pExpr = (SExprNode*)pNode; SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 621a38ea4a88d5b501abd913780204c719c4dfc0..80ba356e8f194f13c21af403bf9be378a0599134 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -24,8 +24,8 @@ #pragma GCC diagnostic ignored "-Wformat-truncation" #endif -int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, - void *(*mallocFp)(int32_t)) = {0}; +int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, + void *(*mallocFp)(int64_t)) = {0}; int32_t (*queryProcessMsgRsp[TDMT_MAX])(void *output, char *msg, int32_t msgSize) = {0}; int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { @@ -67,8 +67,8 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { return TSDB_CODE_SUCCESS; } -int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, - void *(*mallcFp)(int32_t)) { +int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, + void *(*mallcFp)(int64_t)) { SBuildTableInput *pInput = input; if (NULL == input || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -91,7 +91,7 @@ int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } -int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { SBuildUseDBInput *pInput = input; if (NULL == pInput || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -114,7 +114,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms return TSDB_CODE_SUCCESS; } -int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -132,7 +132,7 @@ int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -150,7 +150,7 @@ int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -167,7 +167,7 @@ int32_t queryBuildGetSerVerMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -185,7 +185,7 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -203,8 +203,8 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } -int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, - void *(*mallcFp)(int32_t)) { +int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, + void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -227,7 +227,7 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -245,7 +245,7 @@ int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32 return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -263,7 +263,7 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_ return TSDB_CODE_SUCCESS; } -int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int32_t)) { +int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void *(*mallcFp)(int64_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 626e78b4d34c589d389e2854bdeaed6f0f117bd5..36c6817595d156392a0865958f21ac5b3db1b2f3 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -123,6 +123,7 @@ typedef struct SQWTaskCtx { int32_t execId; int32_t level; + bool queryGotData; bool queryRsped; bool queryEnd; bool queryContinue; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9115c2d3aa5770cfdb5edc3ff448e74b14484386..d2c16a1cb5186ed4dd2e399f78f926c0238656d1 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -10,6 +10,7 @@ #include "tdatablock.h" #include "tmsg.h" #include "tname.h" +#include "tglobal.h" SQWorkerMgmt gQwMgmt = { .lock = 0, @@ -92,6 +93,19 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { return TSDB_CODE_SUCCESS; } +int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t rspCode, bool quickRsp) { + if ((!quickRsp) || QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy) { + if (!ctx->localExec) { + qwBuildAndSendQueryRsp(msgType, &ctx->ctrlConnInfo, rspCode, ctx); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, rspCode, tstrerror(rspCode)); + } + + ctx->queryRsped = true; + } + + return TSDB_CODE_SUCCESS; +} + int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { int32_t code = 0; bool qcontinue = true; @@ -144,7 +158,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { if (numOfResBlock == 0) { QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); } else { - QW_TASK_DLOG("qExecTask done", ""); + QW_TASK_DLOG("qExecTask done, useconds:%" PRIu64, useconds); } dsEndPut(sinkHandle, useconds); @@ -234,6 +248,10 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, int32_t code = 0; SOutputData output = {0}; + if (NULL == ctx->sinkHandle) { + return TSDB_CODE_SUCCESS; + } + *dataLen = 0; while (true) { @@ -407,6 +425,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); } + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase)); + QW_ERR_JRET(ctx->rspCode); + } + if (!ctx->queryRsped) { QW_TASK_ELOG("ready msg has not been processed, phase:%s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); @@ -419,6 +442,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed cause of %s, phase:%s", tstrerror(ctx->rspCode), qwPhaseStr(phase)); + QW_ERR_JRET(ctx->rspCode); + } + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { QW_ERR_JRET(qwDropTask(QW_FPARAMS())); @@ -499,21 +527,17 @@ _return: if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC); + ctx->queryGotData = true; } - if (QW_PHASE_POST_QUERY == phase && ctx) { - if (!ctx->localExec) { - bool rsped = false; - SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; - qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); - qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); - if (!rsped) { - qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); - QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); - } + if (QW_PHASE_POST_QUERY == phase && ctx && !ctx->queryRsped) { + bool rsped = false; + SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; + qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); + qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); + if (!rsped) { + qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false); } - - ctx->queryRsped = true; } if (ctx) { @@ -551,6 +575,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); ctx->ctrlConnInfo = qwMsg->connInfo; + ctx->phase = -1; QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); @@ -604,6 +629,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } + qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); + ctx->level = plan->level; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->sinkHandle, sinkHandle); @@ -619,6 +646,31 @@ _return: input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); + if (ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + void *rsp = NULL; + int32_t dataLen = 0; + SOutputData sOutput = {0}; + QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + + if (rsp) { + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + if (qComplete) { + atomic_store_8((int8_t *)&ctx->queryEnd, true); + } + + qwMsg->connInfo = ctx->dataConnInfo; + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code); + rsp = NULL; + + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, + tstrerror(code), dataLen); + } + } + QW_RET(TSDB_CODE_SUCCESS); } @@ -740,7 +792,9 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { locked = true; // RC WARNING - if (QW_QUERY_RUNNING(ctx)) { + if (-1 == ctx->phase || false == ctx->queryGotData) { + QW_TASK_DLOG_E("task query unfinished"); + } else if (QW_QUERY_RUNNING(ctx)) { atomic_store_8((int8_t *)&ctx->queryContinue, 1); } else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC); diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 0a9913e00447b42567d8f3d7ac74a37247d6a942..d3f29c0e49f38954d4856e826595d17bdb426b9d 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -86,7 +86,7 @@ typedef struct SScalarCtx { } \ } while (0) -int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow); +int32_t sclConvertValueToSclParam(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow); int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); @@ -95,6 +95,11 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_PRECISON(_c) ((_c)->columnData->info.precision) void sclFreeParam(SScalarParam* param); +void doVectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr); +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr); +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr); #ifdef __cplusplus } diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index 1a749c61b3dd369da570cfff367e0e0870cc1714..e633b3922359f00647a792d247454b38599badea 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -20,6 +20,15 @@ extern "C" { #endif +typedef struct SSclVectorConvCtx { + const SScalarParam* pIn; + SScalarParam* pOut; + int32_t startIndex; + int32_t endIndex; + int16_t inType; + int16_t outType; +} SSclVectorConvCtx; + typedef double (*_getDoubleValue_fn_t)(void *src, int32_t index); static FORCE_INLINE double getVectorDoubleValue_TINYINT(void *src, int32_t index) { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 132e0ed2141c81d4a3abca587bf0e2668b1e25f8..81ee173648209c1f490e9e7778ea55c3f063cdf0 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1165,7 +1165,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { SValueNode *valueNode = (SValueNode *)cell->pNode; if (valueNode->node.resType.type != type) { int32_t overflow = 0; - code = doConvertDataType(valueNode, &out, &overflow); + code = sclConvertValueToSclParam(valueNode, &out, &overflow); if (code) { // fltError("convert from %d to %d failed", in.type, out.type); FLT_ERR_RET(code); @@ -1973,7 +1973,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } // todo refactor the convert - int32_t code = doConvertDataType(var, &out, NULL); + int32_t code = sclConvertValueToSclParam(var, &out, NULL); if (code != TSDB_CODE_SUCCESS) { qError("convert value to type[%d] failed", type); return TSDB_CODE_TSC_INVALID_OPERATION; @@ -3792,6 +3792,11 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { return DEAL_RES_CONTINUE; } + if (QUERY_NODE_CASE_WHEN == nodeType(*pNode) || QUERY_NODE_WHEN_THEN == nodeType(*pNode)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { SOperatorNode *node = (SOperatorNode *)*pNode; if (!FLT_IS_COMPARISON_OPERATOR(node->opType)) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index da7e2860364c03dcabcb97bc422f64091d996309..1f09fd4799d7ba87ed545d97ed931c88cb4e8517 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -61,7 +61,7 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara return TSDB_CODE_SUCCESS; } -int32_t doConvertDataType(SValueNode *pValueNode, SScalarParam *out, int32_t *overflow) { +int32_t sclConvertValueToSclParam(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow) { SScalarParam in = {.numOfRows = 1}; int32_t code = sclCreateColumnInfoData(&pValueNode->node.resType, 1, &in); if (code != TSDB_CODE_SUCCESS) { @@ -71,12 +71,34 @@ int32_t doConvertDataType(SValueNode *pValueNode, SScalarParam *out, int32_t *ov colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); colInfoDataEnsureCapacity(out->columnData, 1); - code = vectorConvertImpl(&in, out, overflow); + code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1); sclFreeParam(&in); return code; } +int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) { + SSDataBlock* pb = taosArrayGetP(pBlockList, 0); + SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); + if (NULL == pLeft) { + sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pLeft->numOfRows = pb->info.rows; + + if (pDst->numOfRows < pb->info.rows) { + colInfoDataEnsureCapacity(pDst->columnData, pb->info.rows); + } + + _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(OP_TYPE_ASSIGN); + OperatorFn(pLeft, pSrc, pDst, TSDB_ORDER_ASC); + + taosMemoryFree(pLeft); + + return TSDB_CODE_SUCCESS; +} + int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); if (NULL == pObj) { @@ -110,7 +132,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { } int32_t overflow = 0; - code = doConvertDataType(valueNode, &out, &overflow); + code = sclConvertValueToSclParam(valueNode, &out, &overflow); if (code != TSDB_CODE_SUCCESS) { // sclError("convert data from %d to %d failed", in.type, out.type); SCL_ERR_JRET(code); @@ -178,7 +200,7 @@ void sclFreeRes(SHashObj *res) { } void sclFreeParam(SScalarParam *param) { - if (!param->colAlloced) { + if (NULL == param || !param->colAlloced) { return; } @@ -386,7 +408,8 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t } case QUERY_NODE_FUNCTION: case QUERY_NODE_OPERATOR: - case QUERY_NODE_LOGIC_CONDITION: { + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_CASE_WHEN: { SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { sclError("no result for node, type:%d, node:%p", nodeType(node), node); @@ -540,6 +563,135 @@ _return: SCL_RET(code); } +int32_t sclGetNodeRes(SNode* node, SScalarCtx *ctx, SScalarParam **res) { + if (NULL == node) { + return TSDB_CODE_SUCCESS; + } + + int32_t rowNum = 0; + *res = taosMemoryCalloc(1, sizeof(**res)); + if (NULL == *res) { + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCL_ERR_RET(sclInitParam(node, *res, ctx, &rowNum)); + + return TSDB_CODE_SUCCESS; +} + +int32_t sclWalkCaseWhenList(SScalarCtx *ctx, SNodeList* pList, struct SListCell* pCell, SScalarParam *pCase, SScalarParam *pElse, SScalarParam *pComp, SScalarParam *output, int32_t rowIdx, int32_t totalRows, bool *complete) { + SNode *node = NULL; + SWhenThenNode* pWhenThen = NULL; + SScalarParam *pWhen = NULL; + SScalarParam *pThen = NULL; + int32_t code = 0; + + for (SListCell* cell = pCell; (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) { + pWhenThen = (SWhenThenNode*)node; + + SCL_ERR_RET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_RET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + + vectorCompareImpl(pCase, pWhen, pComp, rowIdx, 1, TSDB_ORDER_ASC, OP_TYPE_EQUAL); + + bool *equal = (bool*)colDataGetData(pComp->columnData, rowIdx); + if (*equal) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0))); + + if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + } + + if (pElse) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0))); + + if (0 == rowIdx && 1 == pCase->numOfRows && 1 == pElse->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + + colDataAppend(output->columnData, rowIdx, NULL, true); + + if (0 == rowIdx && 1 == pCase->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + +_return: + + sclFreeParam(pWhen); + sclFreeParam(pThen); + + SCL_RET(code); +} + +int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList* pList, struct SListCell* pCell, SScalarParam *pElse, SScalarParam *output, + int32_t rowIdx, int32_t totalRows, bool *complete, bool preSingle) { + SNode *node = NULL; + SWhenThenNode* pWhenThen = NULL; + SScalarParam *pWhen = NULL; + SScalarParam *pThen = NULL; + int32_t code = 0; + + for (SListCell* cell = pCell; (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) { + pWhenThen = (SWhenThenNode*)node; + pWhen = NULL; + pThen = NULL; + + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + + bool *whenValue = (bool*)colDataGetData(pWhen->columnData, (pWhen->numOfRows > 1 ? rowIdx : 0)); + + if (*whenValue) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? rowIdx : 0))); + + if (preSingle && 0 == rowIdx && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + + sclFreeParam(pWhen); + sclFreeParam(pThen); + } + + if (pElse) { + colDataAppend(output->columnData, rowIdx, colDataGetData(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0)), colDataIsNull_s(pElse->columnData, (pElse->numOfRows > 1 ? rowIdx : 0))); + + if (preSingle && 0 == rowIdx && 1 == pElse->numOfRows && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + + goto _return; + } + + colDataAppend(output->columnData, rowIdx, NULL, true); + + if (preSingle && 0 == rowIdx && totalRows > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + *complete = true; + } + +_return: + + sclFreeParam(pWhen); + sclFreeParam(pThen); + + SCL_RET(code); +} + int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) { SScalarParam *params = NULL; int32_t rowNum = 0; @@ -698,6 +850,101 @@ _return: SCL_RET(code); } +int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *output) { + int32_t code = 0; + SScalarParam *pCase = NULL; + SScalarParam *pElse = NULL; + SScalarParam *pWhen = NULL; + SScalarParam *pThen = NULL; + SScalarParam comp = {0}; + int32_t rowNum = 1; + bool complete = false; + + if (NULL == node->pWhenThenList || node->pWhenThenList->length <= 0) { + sclError("invalid whenThen list"); + SCL_ERR_RET(TSDB_CODE_INVALID_PARA); + } + + if (ctx->pBlockList) { + SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, 0); + rowNum = pb->info.rows; + output->numOfRows = pb->info.rows; + } + + SCL_ERR_JRET(sclCreateColumnInfoData(&node->node.resType, rowNum, output)); + + SCL_ERR_JRET(sclGetNodeRes(node->pCase, ctx, &pCase)); + SCL_ERR_JRET(sclGetNodeRes(node->pElse, ctx, &pElse)); + + SDataType compType = {0}; + compType.type = TSDB_DATA_TYPE_BOOL; + compType.bytes = tDataTypes[compType.type].bytes; + + SCL_ERR_JRET(sclCreateColumnInfoData(&compType, rowNum, &comp)); + + SNode* tnode = NULL; + SWhenThenNode* pWhenThen = (SWhenThenNode*)node->pWhenThenList->pHead->pNode; + + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); + SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + + if (pCase) { + vectorCompare(pCase, pWhen, &comp, TSDB_ORDER_ASC, OP_TYPE_EQUAL); + + for (int32_t i = 0; i < rowNum; ++i) { + bool *equal = (bool*)colDataGetData(comp.columnData, (comp.numOfRows > 1 ? i : 0)); + if (*equal) { + colDataAppend(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))); + if (0 == i && 1 == pCase->numOfRows && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + break; + } + } else { + SCL_ERR_JRET(sclWalkCaseWhenList(ctx, node->pWhenThenList, node->pWhenThenList->pHead->pNext, pCase, pElse, &comp, output, i, rowNum, &complete)); + if (complete) { + break; + } + } + } + } else { + for (int32_t i = 0; i < rowNum; ++i) { + bool *whenValue = (bool*)colDataGetData(pWhen->columnData, (pWhen->numOfRows > 1 ? i : 0)); + if (*whenValue) { + colDataAppend(output->columnData, i, colDataGetData(pThen->columnData, (pThen->numOfRows > 1 ? i : 0)), colDataIsNull_s(pThen->columnData, (pThen->numOfRows > 1 ? i : 0))); + if (0 == i && 1 == pWhen->numOfRows && 1 == pThen->numOfRows && rowNum > 1) { + SCL_ERR_JRET(sclExtendResRows(output, output, ctx->pBlockList)); + break; + } + } else { + SCL_ERR_JRET(sclWalkWhenList(ctx, node->pWhenThenList, node->pWhenThenList->pHead->pNext, pElse, output, i, rowNum, &complete, (pWhen->numOfRows == 1 && pThen->numOfRows == 1))); + if (complete) { + break; + } + } + } + } + + sclFreeParam(pCase); + sclFreeParam(pElse); + sclFreeParam(&comp); + sclFreeParam(pWhen); + sclFreeParam(pThen); + + return TSDB_CODE_SUCCESS; + +_return: + + sclFreeParam(pCase); + sclFreeParam(pElse); + sclFreeParam(&comp); + sclFreeParam(pWhen); + sclFreeParam(pThen); + sclFreeParam(output); + + SCL_RET(code); +} + + EDealRes sclRewriteNullInOptr(SNode **pNode, SScalarCtx *ctx, EOperatorType opType) { if (opType <= OP_TYPE_CALC_MAX) { SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); @@ -960,9 +1207,66 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } +EDealRes sclRewriteCaseWhen(SNode** pNode, SScalarCtx *ctx) { + SCaseWhenNode *node = (SCaseWhenNode *)*pNode; + + if ((!SCL_IS_CONST_NODE(node->pCase)) || (!SCL_IS_CONST_NODE(node->pElse))) { + return DEAL_RES_CONTINUE; + } + + SNode* tnode = NULL; + FOREACH(tnode, node->pWhenThenList) { + SWhenThenNode* pWhenThen = (SWhenThenNode*)tnode; + if (!SCL_IS_CONST_NODE(pWhenThen->pWhen) || !SCL_IS_CONST_NODE(pWhenThen->pThen)) { + return DEAL_RES_CONTINUE; + } + } + + SScalarParam output = {0}; + ctx->code = sclExecCaseWhen(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + sclFreeParam(&output); + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->translate = true; + + res->node.resType = node->node.resType; + if (colDataIsNull_s(output.columnData, 0)) { + res->isNull = true; + res->node.resType = node->node.resType; + } else { + int32_t type = output.columnData->info.type; + if (IS_VAR_DATA_TYPE(type)) { // todo refactor + res->datum.p = output.columnData->pData; + output.columnData->pData = NULL; + } else { + nodesSetValueNodeValue(res, output.columnData->pData); + } + } + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + sclFreeParam(&output); + return DEAL_RES_CONTINUE; +} + + EDealRes sclConstantsRewriter(SNode **pNode, void *pContext) { SScalarCtx *ctx = (SScalarCtx *)pContext; + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + return sclRewriteOperator(pNode, ctx); + } + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { return sclRewriteFunction(pNode, ctx); } @@ -971,8 +1275,8 @@ EDealRes sclConstantsRewriter(SNode **pNode, void *pContext) { return sclRewriteLogic(pNode, ctx); } - if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { - return sclRewriteOperator(pNode, ctx); + if (QUERY_NODE_CASE_WHEN == nodeType(*pNode)) { + return sclRewriteCaseWhen(pNode, ctx); } return DEAL_RES_CONTINUE; @@ -1082,13 +1386,36 @@ EDealRes sclWalkTarget(SNode *pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } +EDealRes sclWalkCaseWhen(SNode* pNode, SScalarCtx *ctx) { + SCaseWhenNode *node = (SCaseWhenNode *)pNode; + SScalarParam output = {0}; + + ctx->code = sclExecCaseWhen(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + EDealRes sclCalcWalker(SNode *pNode, void *pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || - QUERY_NODE_COLUMN == nodeType(pNode) || QUERY_NODE_LEFT_VALUE == nodeType(pNode)) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) + || QUERY_NODE_COLUMN == nodeType(pNode) || QUERY_NODE_LEFT_VALUE == nodeType(pNode) + || QUERY_NODE_WHEN_THEN == nodeType(pNode)) { return DEAL_RES_CONTINUE; } SScalarCtx *ctx = (SScalarCtx *)pContext; + if (QUERY_NODE_OPERATOR == nodeType(pNode)) { + return sclWalkOperator(pNode, ctx); + } + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { return sclWalkFunction(pNode, ctx); } @@ -1097,38 +1424,19 @@ EDealRes sclCalcWalker(SNode *pNode, void *pContext) { return sclWalkLogic(pNode, ctx); } - if (QUERY_NODE_OPERATOR == nodeType(pNode)) { - return sclWalkOperator(pNode, ctx); - } - if (QUERY_NODE_TARGET == nodeType(pNode)) { return sclWalkTarget(pNode, ctx); } + if (QUERY_NODE_CASE_WHEN == nodeType(pNode)) { + return sclWalkCaseWhen(pNode, ctx); + } + sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } -int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) { - SSDataBlock *pb = taosArrayGetP(pBlockList, 0); - SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); - if (NULL == pLeft) { - sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pLeft->numOfRows = pb->info.rows; - colInfoDataEnsureCapacity(pDst->columnData, pb->info.rows); - - _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(OP_TYPE_ASSIGN); - OperatorFn(pLeft, pSrc, pDst, TSDB_ORDER_ASC); - - taosMemoryFree(pLeft); - - return TSDB_CODE_SUCCESS; -} - int32_t sclCalcConstants(SNode *pNode, bool dual, SNode **pRes) { if (NULL == pNode) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e83fe7ecd06adbb2f9ca022a72c603087b4771a3..f467721248a5949ac2cdf22abb81342c89d531b9 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -367,46 +367,45 @@ static FORCE_INLINE void ncharToVar(char *buf, SScalarParam *pOut, int32_t rowIn taosMemoryFree(t); } -// TODO opt performance, tmp is not needed. -int32_t vectorConvertFromVarData(const SScalarParam *pIn, SScalarParam *pOut, int32_t inType, int32_t outType, - int32_t *overflow) { +//TODO opt performance, tmp is not needed. +int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t* overflow) { bool vton = false; _bufConverteFunc func = NULL; - if (TSDB_DATA_TYPE_BOOL == outType) { + if (TSDB_DATA_TYPE_BOOL == pCtx->outType) { func = varToBool; - } else if (IS_SIGNED_NUMERIC_TYPE(outType)) { + } else if (IS_SIGNED_NUMERIC_TYPE(pCtx->outType)) { func = varToSigned; - } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->outType)) { func = varToUnsigned; - } else if (IS_FLOAT_TYPE(outType)) { + } else if (IS_FLOAT_TYPE(pCtx->outType)) { func = varToFloat; - } else if (outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary - ASSERT(inType == TSDB_DATA_TYPE_NCHAR); + } else if (pCtx->outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary + ASSERT(pCtx->inType == TSDB_DATA_TYPE_NCHAR); func = ncharToVar; vton = true; - } else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar - ASSERT(inType == TSDB_DATA_TYPE_VARCHAR); + } else if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar + ASSERT(pCtx->inType == TSDB_DATA_TYPE_VARCHAR); func = varToNchar; vton = true; - } else if (TSDB_DATA_TYPE_TIMESTAMP == outType) { + } else if (TSDB_DATA_TYPE_TIMESTAMP == pCtx->outType) { func = varToTimestamp; } else { - sclError("invalid convert outType:%d", outType); + sclError("invalid convert outType:%d", pCtx->outType); return TSDB_CODE_QRY_APP_ERROR; } - pOut->numOfRows = pIn->numOfRows; - for (int32_t i = 0; i < pIn->numOfRows; ++i) { - if (IS_HELPER_NULL(pIn->columnData, i)) { - colDataAppendNULL(pOut->columnData, i); + pCtx->pOut->numOfRows = pCtx->pIn->numOfRows; + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { + if (IS_HELPER_NULL(pCtx->pIn->columnData, i)) { + colDataAppendNULL(pCtx->pOut->columnData, i); continue; } - char *data = colDataGetVarData(pIn->columnData, i); - int32_t convertType = inType; - if (inType == TSDB_DATA_TYPE_JSON) { - if (*data == TSDB_DATA_TYPE_NULL) { + char* data = colDataGetVarData(pCtx->pIn->columnData, i); + int32_t convertType = pCtx->inType; + if(pCtx->inType == TSDB_DATA_TYPE_JSON){ + if(*data == TSDB_DATA_TYPE_NULL) { ASSERT(0); } else if (*data == TSDB_DATA_TYPE_NCHAR) { data += CHAR_BYTES; @@ -415,13 +414,13 @@ int32_t vectorConvertFromVarData(const SScalarParam *pIn, SScalarParam *pOut, in terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR; return terrno; } else { - convertNumberToNumber(data + CHAR_BYTES, colDataGetNumData(pOut->columnData, i), *data, outType); + convertNumberToNumber(data+CHAR_BYTES, colDataGetNumData(pCtx->pOut->columnData, i), *data, pCtx->outType); continue; } } - int32_t bufSize = pIn->columnData->info.bytes; - char *tmp = taosMemoryMalloc(varDataTLen(data)); - if (!tmp) { + int32_t bufSize = pCtx->pIn->columnData->info.bytes; + char *tmp = taosMemoryMalloc(varDataTLen(data)); + if(!tmp){ sclError("out of memory in vectorConvertFromVarData"); return TSDB_CODE_OUT_OF_MEMORY; } @@ -444,8 +443,8 @@ int32_t vectorConvertFromVarData(const SScalarParam *pIn, SScalarParam *pOut, in tmp[len] = 0; } } - - (*func)(tmp, pOut, i, overflow); + + (*func)(tmp, pCtx->pOut, i, overflow); taosMemoryFreeClear(tmp); } @@ -581,64 +580,64 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t return true; } -int32_t vectorConvertToVarData(const SScalarParam *pIn, SScalarParam *pOut, int16_t inType, int16_t outType) { - SColumnInfoData *pInputCol = pIn->columnData; - SColumnInfoData *pOutputCol = pOut->columnData; - char tmp[128] = {0}; +int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { + SColumnInfoData* pInputCol = pCtx->pIn->columnData; + SColumnInfoData* pOutputCol = pCtx->pOut->columnData; + char tmp[128] = {0}; - if (IS_SIGNED_NUMERIC_TYPE(inType) || inType == TSDB_DATA_TYPE_BOOL || inType == TSDB_DATA_TYPE_TIMESTAMP) { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inType) || pCtx->inType == TSDB_DATA_TYPE_BOOL || pCtx->inType == TSDB_DATA_TYPE_TIMESTAMP) { + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int64_t, pCtx->inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value); varDataLen(tmp) = len; - if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i, NULL); + if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pCtx->pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } } - } else if (IS_UNSIGNED_NUMERIC_TYPE(inType)) { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inType)) { + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint64_t, pCtx->inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value); varDataLen(tmp) = len; - if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i, NULL); + if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pCtx->pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } } - } else if (IS_FLOAT_TYPE(inType)) { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + } else if (IS_FLOAT_TYPE(pCtx->inType)) { + for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, double, pCtx->inType, colDataGetData(pInputCol, i)); int32_t len = sprintf(varDataVal(tmp), "%lf", value); varDataLen(tmp) = len; - if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i, NULL); + if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pCtx->pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } } } else { - sclError("not supported input type:%d", inType); + sclError("not supported input type:%d", pCtx->inType); return TSDB_CODE_QRY_APP_ERROR; } @@ -646,34 +645,35 @@ int32_t vectorConvertToVarData(const SScalarParam *pIn, SScalarParam *pOut, int1 } // TODO opt performance -int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow) { - SColumnInfoData *pInputCol = pIn->columnData; - SColumnInfoData *pOutputCol = pOut->columnData; +int32_t vectorConvertSingleColImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* overflow, int32_t startIndex, int32_t numOfRows) { + SColumnInfoData* pInputCol = pIn->columnData; + SColumnInfoData* pOutputCol = pOut->columnData; if (NULL == pInputCol) { sclError("input column is NULL, hashFilter %p", pIn->pHashFilter); return TSDB_CODE_APP_ERROR; } - int16_t inType = pInputCol->info.type; - int16_t outType = pOutputCol->info.type; + int32_t rstart = startIndex >= 0 ? startIndex : 0; + int32_t rend = numOfRows > 0 ? rstart + numOfRows - 1 : rstart + pIn->numOfRows - 1; + SSclVectorConvCtx cCtx = {pIn, pOut, rstart, rend, pInputCol->info.type, pOutputCol->info.type}; - if (IS_VAR_DATA_TYPE(inType)) { - return vectorConvertFromVarData(pIn, pOut, inType, outType, overflow); + if (IS_VAR_DATA_TYPE(cCtx.inType)) { + return vectorConvertFromVarData(&cCtx, overflow); } if (overflow) { ASSERT(1 == pIn->numOfRows); pOut->numOfRows = 0; - - if (IS_SIGNED_NUMERIC_TYPE(outType)) { - int64_t minValue = tDataTypes[outType].minValue; - int64_t maxValue = tDataTypes[outType].maxValue; - + + if (IS_SIGNED_NUMERIC_TYPE(cCtx.outType)) { + int64_t minValue = tDataTypes[cCtx.outType].minValue; + int64_t maxValue = tDataTypes[cCtx.outType].maxValue; + double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0)); - + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, 0)); + if (value > maxValue) { *overflow = 1; return TSDB_CODE_SUCCESS; @@ -683,13 +683,13 @@ int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t * } else { *overflow = 0; } - } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { - uint64_t minValue = (uint64_t)tDataTypes[outType].minValue; - uint64_t maxValue = (uint64_t)tDataTypes[outType].maxValue; - + } else if (IS_UNSIGNED_NUMERIC_TYPE(cCtx.outType)) { + uint64_t minValue = (uint64_t)tDataTypes[cCtx.outType].minValue; + uint64_t maxValue = (uint64_t)tDataTypes[cCtx.outType].maxValue; + double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0)); - + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, 0)); + if (value > maxValue) { *overflow = 1; return TSDB_CODE_SUCCESS; @@ -703,157 +703,157 @@ int32_t vectorConvertImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t * } pOut->numOfRows = pIn->numOfRows; - switch (outType) { + switch (cCtx.outType) { case TSDB_DATA_TYPE_BOOL: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } bool value = 0; - GET_TYPED_DATA(value, bool, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, bool, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt8(pOutputCol, i, (int8_t *)&value); } break; } case TSDB_DATA_TYPE_TINYINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int8_t value = 0; - GET_TYPED_DATA(value, int8_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int8_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt8(pOutputCol, i, (int8_t *)&value); } break; } - case TSDB_DATA_TYPE_SMALLINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_SMALLINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int16_t value = 0; - GET_TYPED_DATA(value, int16_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int16_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt16(pOutputCol, i, (int16_t *)&value); } break; } - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_INT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int32_t value = 0; - GET_TYPED_DATA(value, int32_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int32_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt32(pOutputCol, i, (int32_t *)&value); } break; } case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, int64_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt64(pOutputCol, i, (int64_t *)&value); } break; } - case TSDB_DATA_TYPE_UTINYINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_UTINYINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint8_t value = 0; - GET_TYPED_DATA(value, uint8_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint8_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt8(pOutputCol, i, (int8_t *)&value); } break; } - case TSDB_DATA_TYPE_USMALLINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_USMALLINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint16_t value = 0; - GET_TYPED_DATA(value, uint16_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint16_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt16(pOutputCol, i, (int16_t *)&value); } break; } - case TSDB_DATA_TYPE_UINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_UINT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint32_t value = 0; - GET_TYPED_DATA(value, uint32_t, inType, colDataGetData(pInputCol, i)); + GET_TYPED_DATA(value, uint32_t, cCtx.inType, colDataGetData(pInputCol, i)); colDataAppendInt32(pOutputCol, i, (int32_t *)&value); } break; } case TSDB_DATA_TYPE_UBIGINT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); - colDataAppendInt64(pOutputCol, i, (int64_t *)&value); + GET_TYPED_DATA(value, uint64_t, cCtx.inType, colDataGetData(pInputCol, i)); + colDataAppendInt64(pOutputCol, i, (int64_t*)&value); } break; } - case TSDB_DATA_TYPE_FLOAT: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + case TSDB_DATA_TYPE_FLOAT:{ + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } float value = 0; - GET_TYPED_DATA(value, float, inType, colDataGetData(pInputCol, i)); - colDataAppendFloat(pOutputCol, i, (float *)&value); + GET_TYPED_DATA(value, float, cCtx.inType, colDataGetData(pInputCol, i)); + colDataAppendFloat(pOutputCol, i, (float*)&value); } break; } case TSDB_DATA_TYPE_DOUBLE: { - for (int32_t i = 0; i < pIn->numOfRows; ++i) { + for (int32_t i = cCtx.startIndex; i <= cCtx.endIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { colDataAppendNULL(pOutputCol, i); continue; } double value = 0; - GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); - colDataAppendDouble(pOutputCol, i, (double *)&value); + GET_TYPED_DATA(value, double, cCtx.inType, colDataGetData(pInputCol, i)); + colDataAppendDouble(pOutputCol, i, (double*)&value); } break; } case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { - return vectorConvertToVarData(pIn, pOut, inType, outType); + return vectorConvertToVarData(&cCtx); } default: - sclError("invalid convert output type:%d", outType); + sclError("invalid convert output type:%d", cCtx.outType); return TSDB_CODE_QRY_APP_ERROR; } @@ -894,7 +894,7 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) { return gConvertTypes[type2][type1]; } -int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int32_t type) { +int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows) { SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; output->numOfRows = input->numOfRows; @@ -903,17 +903,16 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3 return TSDB_CODE_OUT_OF_MEMORY; } - code = vectorConvertImpl(input, output, NULL); + code = vectorConvertSingleColImpl(input, output, NULL, startIndex, numOfRows); if (code) { - // taosMemoryFreeClear(paramOut1->data); return code; } return TSDB_CODE_SUCCESS; } -int32_t vectorConvert(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pLeftOut, SScalarParam *pRightOut) { - int32_t leftType = GET_PARAM_TYPE(pLeft); +int32_t vectorConvertCols(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut, int32_t startIndex, int32_t numOfRows) { + int32_t leftType = GET_PARAM_TYPE(pLeft); int32_t rightType = GET_PARAM_TYPE(pRight); if (leftType == rightType) { return TSDB_CODE_SUCCESS; @@ -941,14 +940,14 @@ int32_t vectorConvert(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p } if (type != GET_PARAM_TYPE(param1)) { - code = vectorConvertScalarParam(param1, paramOut1, type); + code = vectorConvertSingleCol(param1, paramOut1, type, startIndex, numOfRows); if (code) { return code; } } if (type != GET_PARAM_TYPE(param2)) { - code = vectorConvertScalarParam(param2, paramOut2, type); + code = vectorConvertSingleCol(param2, paramOut2, type, startIndex, numOfRows); if (code) { return code; } @@ -962,31 +961,6 @@ enum { VECTOR_UN_CONVERT = 0x2, }; -static int32_t doConvertHelper(SScalarParam *pDest, int32_t *convert, const SScalarParam *pParam, int32_t type) { - SColumnInfoData *pCol = pParam->columnData; - - if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) { - pDest->numOfRows = pParam->numOfRows; - - SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - int32_t code = sclCreateColumnInfoData(&t, pParam->numOfRows, pDest); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = vectorConvertImpl(pParam, pDest, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - *convert = VECTOR_DO_CONVERT; - } else { - *convert = VECTOR_UN_CONVERT; - } - - return TSDB_CODE_SUCCESS; -} - // TODO not correct for descending order scan static void vectorMathAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t i) { @@ -1030,20 +1004,25 @@ static void vectorMathTsAddHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pR } } -static SColumnInfoData *doVectorConvert(SScalarParam *pInput, int32_t *doConvert) { - SScalarParam convertParam = {0}; +static SColumnInfoData* vectorConvertVarToDouble(SScalarParam* pInput, int32_t* converted) { + SScalarParam output = {0}; + SColumnInfoData* pCol = pInput->columnData; - int32_t code = doConvertHelper(&convertParam, doConvert, pInput, TSDB_DATA_TYPE_DOUBLE); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; - } + if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) { + int32_t code = vectorConvertSingleCol(pInput, &output, TSDB_DATA_TYPE_DOUBLE, -1, -1); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } - if (*doConvert == VECTOR_DO_CONVERT) { - return convertParam.columnData; - } else { - return pInput->columnData; + *converted = VECTOR_DO_CONVERT; + + return output.columnData; } + + *converted = VECTOR_UN_CONVERT; + + return pInput->columnData; } static void doReleaseVec(SColumnInfoData *pCol, int32_t type) { @@ -1061,9 +1040,9 @@ void vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pRight))) || (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pLeft))) || @@ -1168,9 +1147,9 @@ void vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BIGINT) || (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && @@ -1246,9 +1225,9 @@ void vectorMathMultiply(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); @@ -1279,9 +1258,9 @@ void vectorMathDivide(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); @@ -1333,9 +1312,9 @@ void vectorMathRemainder(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); @@ -1412,8 +1391,8 @@ void vectorMathMinus(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : (pLeft->numOfRows - 1); int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); + int32_t leftConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeftCol->info.type); @@ -1525,9 +1504,9 @@ void vectorBitAnd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); @@ -1579,9 +1558,9 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int32_t leftConvert = 0, rightConvert = 0; - SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); - SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); + int32_t leftConvert = 0, rightConvert = 0; + SColumnInfoData *pLeftCol = vectorConvertVarToDouble(pLeft, &leftConvert); + SColumnInfoData *pRightCol = vectorConvertVarToDouble(pRight, &rightConvert); _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); @@ -1605,8 +1584,8 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, doReleaseVec(pRightCol, rightConvert); } -int32_t doVectorCompareImpl(int32_t numOfRows, SScalarParam *pOut, int32_t startIndex, int32_t step, __compar_fn_t fp, - SScalarParam *pLeft, SScalarParam *pRight, int32_t optr) { +int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t step, __compar_fn_t fp, int32_t optr) { int32_t num = 0; for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { @@ -1659,20 +1638,29 @@ int32_t doVectorCompareImpl(int32_t numOfRows, SScalarParam *pOut, int32_t start return num; } -void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; +void doVectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr) { + int32_t i = 0; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int32_t lType = GET_PARAM_TYPE(pLeft); int32_t rType = GET_PARAM_TYPE(pRight); __compar_fn_t fp = NULL; - + int32_t compRows = 0; + if (lType == rType) { fp = filterGetCompFunc(lType, optr); } else { fp = filterGetCompFuncEx(lType, rType, optr); } - pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + if (startIndex < 0) { + i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; + pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + compRows = pOut->numOfRows; + } else { + compRows = startIndex + numOfRows; + i = startIndex; + } if (pRight->pHashFilter != NULL) { for (; i >= 0 && i < pLeft->numOfRows; i += step) { @@ -1690,13 +1678,14 @@ void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam * } } } else { // normal compare - pOut->numOfQualified = doVectorCompareImpl(pOut->numOfRows, pOut, i, step, fp, pLeft, pRight, optr); + pOut->numOfQualified = doVectorCompareImpl(pLeft, pRight, pOut, i, compRows, step, fp, optr); } } -void vectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { - SScalarParam pLeftOut = {0}; - SScalarParam pRightOut = {0}; +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, + int32_t _ord, int32_t optr) { + SScalarParam pLeftOut = {0}; + SScalarParam pRightOut = {0}; SScalarParam *param1 = NULL; SScalarParam *param2 = NULL; @@ -1704,7 +1693,7 @@ void vectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut param1 = pLeft; param2 = pRight; } else { - vectorConvert(pLeft, pRight, &pLeftOut, &pRightOut); + vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows); if (pLeftOut.columnData != NULL) { param1 = &pLeftOut; @@ -1719,12 +1708,17 @@ void vectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut } } - vectorCompareImpl(param1, param2, pOut, _ord, optr); + doVectorCompare(param1, param2, pOut, startIndex, numOfRows, _ord, optr); + sclFreeParam(&pLeftOut); sclFreeParam(&pRightOut); } -void vectorGreater(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { + vectorCompareImpl(pLeft, pRight, pOut, -1, -1, _ord, optr); +} + +void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_GREATER_THAN); } @@ -1788,10 +1782,10 @@ void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut pOut->numOfRows = pLeft->numOfRows; } -void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { - vectorConvertImpl(pLeft, pOut, NULL); - for (int32_t i = 0; i < pOut->numOfRows; ++i) { - if (colDataIsNull_s(pOut->columnData, i)) { +void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorConvertSingleColImpl(pLeft, pOut, NULL, -1, -1); + for(int32_t i = 0; i < pOut->numOfRows; ++i) { + if(colDataIsNull_s(pOut->columnData, i)) { int8_t v = 0; colDataAppendInt8(pOut->columnData, i, &v); colDataSetNotNull_f(pOut->columnData->nullbitmap, i); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 7ced4f626c413d7f208d524e833c3b2626678751..1a88992840cd649d2538f66e19d86a96df34c594 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -20,13 +20,13 @@ extern "C" { #endif +#include "command.h" #include "os.h" -#include "tarray.h" #include "planner.h" #include "scheduler.h" +#include "tarray.h" #include "thash.h" #include "trpc.h" -#include "command.h" enum { SCH_READ = 1, @@ -51,20 +51,20 @@ typedef enum { SCH_ALL, } SCH_POLICY; -#define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 -#define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 +#define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 +#define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 #define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT -#define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ -#define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 +#define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ +#define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 #define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000 -#define SCH_MAX_TASK_TIMEOUT_USEC 60000000 -#define SCH_DEFAULT_MAX_RETRY_NUM 6 -#define SCH_MIN_AYSNC_EXEC_NUM 3 +#define SCH_MAX_TASK_TIMEOUT_USEC 60000000 +#define SCH_DEFAULT_MAX_RETRY_NUM 6 +#define SCH_MIN_AYSNC_EXEC_NUM 3 typedef struct SSchDebug { - bool lockEnable; - bool apiEnable; + bool lockEnable; + bool apiEnable; } SSchDebug; typedef struct SSchTrans { @@ -80,7 +80,6 @@ typedef struct SSchHbTrans { } SSchHbTrans; typedef struct SSchApiStat { - #if defined(WINDOWS) || defined(_TD_DARWIN_64) size_t avoidCompilationErrors; #endif @@ -88,7 +87,6 @@ typedef struct SSchApiStat { } SSchApiStat; typedef struct SSchRuntimeStat { - #if defined(WINDOWS) || defined(_TD_DARWIN_64) size_t avoidCompilationErrors; #endif @@ -96,7 +94,6 @@ typedef struct SSchRuntimeStat { } SSchRuntimeStat; typedef struct SSchJobStat { - #if defined(WINDOWS) || defined(_TD_DARWIN_64) size_t avoidCompilationErrors; #endif @@ -104,17 +101,17 @@ typedef struct SSchJobStat { } SSchJobStat; typedef struct SSchStat { - SSchApiStat api; - SSchRuntimeStat runtime; - SSchJobStat job; + SSchApiStat api; + SSchRuntimeStat runtime; + SSchJobStat job; } SSchStat; typedef struct SSchResInfo { - SExecResult* execRes; - void** fetchRes; - schedulerExecFp execFp; - schedulerFetchFp fetchFp; - void* cbParam; + SExecResult *execRes; + void **fetchRes; + schedulerExecFp execFp; + schedulerFetchFp fetchFp; + void *cbParam; } SSchResInfo; typedef struct SSchOpEvent { @@ -123,9 +120,9 @@ typedef struct SSchOpEvent { SSchedulerReq *pReq; } SSchOpEvent; -typedef int32_t (*schStatusEnterFp)(void* pHandle, void* pParam); -typedef int32_t (*schStatusLeaveFp)(void* pHandle, void* pParam); -typedef int32_t (*schStatusEventFp)(void* pHandle, void* pParam, void* pEvent); +typedef int32_t (*schStatusEnterFp)(void *pHandle, void *pParam); +typedef int32_t (*schStatusLeaveFp)(void *pHandle, void *pParam); +typedef int32_t (*schStatusEventFp)(void *pHandle, void *pParam, void *pEvent); typedef struct SSchStatusFps { EJobTaskType status; @@ -142,16 +139,16 @@ typedef struct SSchedulerCfg { } SSchedulerCfg; typedef struct SSchedulerMgmt { - uint64_t taskId; // sequential taksId - uint64_t sId; // schedulerId - SSchedulerCfg cfg; - bool exit; - int32_t jobRef; - int32_t jobNum; - SSchStat stat; - SRWLatch hbLock; - SHashObj *hbConnections; - void *queryMgmt; + uint64_t taskId; // sequential taksId + uint64_t sId; // schedulerId + SSchedulerCfg cfg; + bool exit; + int32_t jobRef; + int32_t jobNum; + SSchStat stat; + SRWLatch hbLock; + SHashObj *hbConnections; + void *queryMgmt; } SSchedulerMgmt; typedef struct SSchCallbackParamHeader { @@ -174,11 +171,11 @@ typedef struct SSchHbCallbackParam { } SSchHbCallbackParam; typedef struct SSchFlowControl { - SRWLatch lock; - bool sorted; - int32_t tableNumSum; - uint32_t execTaskNum; - SArray *taskList; // Element is SSchTask* + SRWLatch lock; + bool sorted; + int32_t tableNumSum; + uint32_t execTaskNum; + SArray *taskList; // Element is SSchTask* } SSchFlowControl; typedef struct SSchNodeInfo { @@ -187,50 +184,50 @@ typedef struct SSchNodeInfo { } SSchNodeInfo; typedef struct SSchLevel { - int32_t level; - int8_t status; - SRWLatch lock; - int32_t taskFailed; - int32_t taskSucceed; - int32_t taskNum; - int32_t taskLaunchedNum; - int32_t taskDoneNum; - SArray *subTasks; // Element is SSchTask + int32_t level; + int8_t status; + SRWLatch lock; + int32_t taskFailed; + int32_t taskSucceed; + int32_t taskNum; + int32_t taskLaunchedNum; + int32_t taskDoneNum; + SArray *subTasks; // Element is SSchTask } SSchLevel; typedef struct SSchTaskProfile { - int64_t startTs; - SArray* execTime; - int64_t waitTime; - int64_t endTs; + int64_t startTs; + SArray *execTime; + int64_t waitTime; + int64_t endTs; } SSchTaskProfile; typedef struct SSchTask { - uint64_t taskId; // task id - SRWLatch lock; // task reentrant lock - int32_t maxExecTimes; // task max exec times - int32_t maxRetryTimes; // task max retry times - int32_t retryTimes; // task retry times - bool waitRetry; // wait for retry - int32_t execId; // task current execute index - SSchLevel *level; // level - SRWLatch planLock; // task update plan lock - SSubplan *plan; // subplan - char *msg; // operator tree - int32_t msgLen; // msg length - int8_t status; // task status - int32_t lastMsgType; // last sent msg type - int64_t timeoutUsec; // task timeout useconds before reschedule - SQueryNodeAddr succeedAddr; // task executed success node address - int8_t candidateIdx; // current try condidation index - SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr - SHashObj *execNodes; // all tried node for current task, element is SSchNodeInfo - SSchTaskProfile profile; // task execution profile - int32_t childReady; // child task ready number - SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* - SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* - void* handle; // task send handle - bool registerdHb; // registered in hb + uint64_t taskId; // task id + SRWLatch lock; // task reentrant lock + int32_t maxExecTimes; // task max exec times + int32_t maxRetryTimes; // task max retry times + int32_t retryTimes; // task retry times + bool waitRetry; // wait for retry + int32_t execId; // task current execute index + SSchLevel *level; // level + SRWLatch planLock; // task update plan lock + SSubplan *plan; // subplan + char *msg; // operator tree + int32_t msgLen; // msg length + int8_t status; // task status + int32_t lastMsgType; // last sent msg type + int64_t timeoutUsec; // task timeout useconds before reschedule + SQueryNodeAddr succeedAddr; // task executed success node address + int8_t candidateIdx; // current try condidation index + SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr + SHashObj *execNodes; // all tried node for current task, element is SSchNodeInfo + SSchTaskProfile profile; // task execution profile + int32_t childReady; // child task ready number + SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* + SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* + void *handle; // task send handle + bool registerdHb; // registered in hb } SSchTask; typedef struct SSchJobAttr { @@ -243,46 +240,46 @@ typedef struct SSchJobAttr { } SSchJobAttr; typedef struct { - int32_t op; - SRWLatch lock; - bool syncReq; + int32_t op; + SRWLatch lock; + bool syncReq; } SSchOpStatus; typedef struct SSchJob { - int64_t refId; - uint64_t queryId; - SSchJobAttr attr; - int32_t levelNum; - int32_t taskNum; - SRequestConnInfo conn; - SArray *nodeList; // qnode/vnode list, SArray - SArray *levels; // starting from 0. SArray - SQueryPlan *pDag; - int64_t allocatorRefId; - - SArray *dataSrcTasks; // SArray - int32_t levelIdx; - SEpSet dataSrcEps; - SHashObj *taskList; - SHashObj *execTasks; // executing and executed tasks, key:taskid, value:SQueryTask* - SHashObj *flowCtrl; // key is ep, element is SSchFlowControl - - SExplainCtx *explainCtx; - int8_t status; - SQueryNodeAddr resNode; - tsem_t rspSem; - SSchOpStatus opStatus; - schedulerChkKillFp chkKillFp; - void* chkKillParam; - SSchTask *fetchTask; - int32_t errCode; - SRWLatch resLock; - SExecResult execRes; - void *fetchRes; //TODO free it or not - bool fetched; - int32_t resNumOfRows; - SSchResInfo userRes; - char *sql; + int64_t refId; + uint64_t queryId; + SSchJobAttr attr; + int32_t levelNum; + int32_t taskNum; + SRequestConnInfo conn; + SArray *nodeList; // qnode/vnode list, SArray + SArray *levels; // starting from 0. SArray + SQueryPlan *pDag; + int64_t allocatorRefId; + + SArray *dataSrcTasks; // SArray + int32_t levelIdx; + SEpSet dataSrcEps; + SHashObj *taskList; + SHashObj *execTasks; // executing and executed tasks, key:taskid, value:SQueryTask* + SHashObj *flowCtrl; // key is ep, element is SSchFlowControl + + SExplainCtx *explainCtx; + int8_t status; + SQueryNodeAddr resNode; + tsem_t rspSem; + SSchOpStatus opStatus; + schedulerChkKillFp chkKillFp; + void *chkKillParam; + SSchTask *fetchTask; + int32_t errCode; + SRWLatch resLock; + SExecResult execRes; + void *fetchRes; // TODO free it or not + bool fetched; + int32_t resNumOfRows; + SSchResInfo userRes; + char *sql; SQueryProfileSummary summary; } SSchJob; @@ -294,226 +291,282 @@ typedef struct SSchTaskCtx { extern SSchedulerMgmt schMgmt; -#define SCH_TASK_TIMEOUT(_task) ((taosGetTimestampUs() - *(int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId)) > (_task)->timeoutUsec) +#define SCH_TASK_TIMEOUT(_task) \ + ((taosGetTimestampUs() - *(int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId)) > (_task)->timeoutUsec) #define SCH_TASK_READY_FOR_LAUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children)) -#define SCH_LOCK_TASK(_task) SCH_LOCK(SCH_WRITE, &(_task)->lock) +#define SCH_LOCK_TASK(_task) SCH_LOCK(SCH_WRITE, &(_task)->lock) #define SCH_UNLOCK_TASK(_task) SCH_UNLOCK(SCH_WRITE, &(_task)->lock) -#define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) +#define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) #define SCH_TASK_EID(_task) ((_task) ? (_task)->execId : -1) #define SCH_IS_DATA_BIND_QRY_TASK(task) ((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) -#define SCH_IS_DATA_BIND_TASK(task) (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY)) +#define SCH_IS_DATA_BIND_TASK(task) \ + (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY)) #define SCH_IS_LEAF_TASK(_job, _task) (((_task)->level->level + 1) == (_job)->levelNum) -#define SCH_IS_DATA_MERGE_TASK(task) (!SCH_IS_DATA_BIND_TASK(task)) -#define SCH_IS_LOCAL_EXEC_TASK(_job, _task) ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && (!SCH_IS_DATA_BIND_QRY_TASK(_task))) +#define SCH_IS_DATA_MERGE_TASK(task) (!SCH_IS_DATA_BIND_TASK(task)) +#define SCH_IS_LOCAL_EXEC_TASK(_job, _task) \ + ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \ + (!SCH_IS_DATA_BIND_QRY_TASK(_task))) #define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) -#define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) +#define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) #define SCH_GET_TASK_STATUS_STR(task) jobTaskStatusStr(SCH_GET_TASK_STATUS(task)) -#define SCH_GET_TASK_HANDLE(_task) ((_task) ? (_task)->handle : NULL) +#define SCH_GET_TASK_HANDLE(_task) ((_task) ? (_task)->handle : NULL) #define SCH_SET_TASK_HANDLE(_task, _handle) ((_task)->handle = (_handle)) #define SCH_SET_JOB_STATUS(job, st) atomic_store_8(&(job)->status, st) -#define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status) +#define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status) #define SCH_GET_JOB_STATUS_STR(job) jobTaskStatusStr(SCH_GET_JOB_STATUS(job)) #define SCH_JOB_IN_SYNC_OP(job) ((job)->opStatus.op && (job)->opStatus.syncReq) -#define SCH_JOB_IN_ASYNC_EXEC_OP(job) ((SCH_OP_EXEC == atomic_val_compare_exchange_32(&(job)->opStatus.op, SCH_OP_EXEC, SCH_OP_NULL)) && (!(job)->opStatus.syncReq)) -#define SCH_JOB_IN_ASYNC_FETCH_OP(job) ((SCH_OP_FETCH == atomic_val_compare_exchange_32(&(job)->opStatus.op, SCH_OP_FETCH, SCH_OP_NULL)) && (!(job)->opStatus.syncReq)) +#define SCH_JOB_IN_ASYNC_EXEC_OP(job) \ + ((SCH_OP_EXEC == atomic_val_compare_exchange_32(&(job)->opStatus.op, SCH_OP_EXEC, SCH_OP_NULL)) && \ + (!(job)->opStatus.syncReq)) +#define SCH_JOB_IN_ASYNC_FETCH_OP(job) \ + ((SCH_OP_FETCH == atomic_val_compare_exchange_32(&(job)->opStatus.op, SCH_OP_FETCH, SCH_OP_NULL)) && \ + (!(job)->opStatus.syncReq)) #define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true -#define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl) -#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_BIND_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) -#define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) +#define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl) +#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) \ + (SCH_IS_DATA_BIND_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) +#define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) #define SCH_TASK_NEED_FETCH(_task) ((_task)->plan->subplanType != SUBPLAN_TYPE_MODIFY) -#define SCH_SET_JOB_TYPE(_job, type) do { if ((type) != SUBPLAN_TYPE_MODIFY) { (_job)->attr.queryJob = true; } else { (_job)->attr.insertJob = true; } } while (0) -#define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) -#define SCH_IS_INSERT_JOB(_job) ((_job)->attr.insertJob) +#define SCH_SET_JOB_TYPE(_job, type) \ + do { \ + if ((type) != SUBPLAN_TYPE_MODIFY) { \ + (_job)->attr.queryJob = true; \ + } else { \ + (_job)->attr.insertJob = true; \ + } \ + } while (0) +#define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) +#define SCH_IS_INSERT_JOB(_job) ((_job)->attr.insertJob) #define SCH_JOB_NEED_FETCH(_job) ((_job)->attr.needFetch) -#define SCH_JOB_NEED_WAIT(_job) (!SCH_IS_QUERY_JOB(_job)) -#define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job)) +#define SCH_JOB_NEED_WAIT(_job) (!SCH_IS_QUERY_JOB(_job)) +#define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job)) #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) -#define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) -#define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)))) -#define SCH_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_LINK_BROKEN || (_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) -#define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) (SCH_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen)))) -#define SCH_NEED_RETRY(_msgType, _code) ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) +#define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) +#define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) \ + (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)))) +#define SCH_REDIRECT_MSGTYPE(_msgType) \ + ((_msgType) == TDMT_SCH_LINK_BROKEN || (_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || \ + (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) +#define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) \ + (SCH_REDIRECT_MSGTYPE(_msgType) && \ + (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen)))) +#define SCH_NEED_RETRY(_msgType, _code) \ + ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum) -#define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) -#define SCH_SWITCH_EPSET(_addr) ((_addr)->epSet.inUse = ((_addr)->epSet.inUse + 1) % (_addr)->epSet.numOfEps) -#define SCH_TASK_NUM_OF_EPS(_addr) ((_addr)->epSet.numOfEps) - -#define SCH_LOG_TASK_START_TS(_task) \ - do { \ - int64_t us = taosGetTimestampUs(); \ - taosArrayPush((_task)->profile.execTime, &us); \ - if (0 == (_task)->execId) { \ - (_task)->profile.startTs = us; \ - } \ - } while (0) - -#define SCH_LOG_TASK_WAIT_TS(_task) \ - do { \ - int64_t us = taosGetTimestampUs(); \ - (_task)->profile.waitTime += us - *(int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ - } while (0) - - -#define SCH_LOG_TASK_END_TS(_task) \ - do { \ - int64_t us = taosGetTimestampUs(); \ - int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ +#define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) +#define SCH_SWITCH_EPSET(_addr) ((_addr)->epSet.inUse = ((_addr)->epSet.inUse + 1) % (_addr)->epSet.numOfEps) +#define SCH_TASK_NUM_OF_EPS(_addr) ((_addr)->epSet.numOfEps) + +#define SCH_LOG_TASK_START_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ + taosArrayPush((_task)->profile.execTime, &us); \ + if (0 == (_task)->execId) { \ + (_task)->profile.startTs = us; \ + } \ + } while (0) + +#define SCH_LOG_TASK_WAIT_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ + (_task)->profile.waitTime += us - *(int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + } while (0) + +#define SCH_LOG_TASK_END_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ + int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ int64_t *startts = taosArrayGet((_task)->profile.execTime, (_task)->execId); \ - *startts = us - *startts; \ - (_task)->profile.endTs = us; \ - } while (0) - + *startts = us - *startts; \ + (_task)->profile.endTs = us; \ + } while (0) #define SCH_JOB_ELOG(param, ...) qError("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) #define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) -#define SCH_TASK_ELOG(param, ...) \ - qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) -#define SCH_TASK_DLOG(param, ...) \ - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) -#define SCH_TASK_TLOG(param, ...) \ - qTrace("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) -#define SCH_TASK_DLOGL(param, ...) \ - qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) -#define SCH_TASK_WLOG(param, ...) \ - qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) - -#define SCH_SET_ERRNO(_err) do { if (TSDB_CODE_SCH_IGNORE_ERROR != (_err)) { terrno = (_err); } } while (0) -#define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { SCH_SET_ERRNO(_code); return _code; } } while (0) -#define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { SCH_SET_ERRNO(_code); } return _code; } while (0) -#define SCH_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { SCH_SET_ERRNO(code); goto _return; } } while (0) - -#define SCH_LOCK_DEBUG(...) do { if (gSCHDebug.lockEnable) { qDebug(__VA_ARGS__); } } while (0) +#define SCH_TASK_ELOG(param, ...) \ + qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + __VA_ARGS__) +#define SCH_TASK_DLOG(param, ...) \ + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + __VA_ARGS__) +#define SCH_TASK_TLOG(param, ...) \ + qTrace("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + __VA_ARGS__) +#define SCH_TASK_DLOGL(param, ...) \ + qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + __VA_ARGS__) +#define SCH_TASK_WLOG(param, ...) \ + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask), \ + __VA_ARGS__) + +#define SCH_SET_ERRNO(_err) \ + do { \ + if (TSDB_CODE_SCH_IGNORE_ERROR != (_err)) { \ + terrno = (_err); \ + } \ + } while (0) +#define SCH_ERR_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + SCH_SET_ERRNO(_code); \ + return _code; \ + } \ + } while (0) +#define SCH_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + SCH_SET_ERRNO(_code); \ + } \ + return _code; \ + } while (0) +#define SCH_ERR_JRET(c) \ + do { \ + code = c; \ + if (code != TSDB_CODE_SUCCESS) { \ + SCH_SET_ERRNO(code); \ + goto _return; \ + } \ + } while (0) + +#define SCH_LOCK_DEBUG(...) \ + do { \ + if (gSCHDebug.lockEnable) { \ + qDebug(__VA_ARGS__); \ + } \ + } while (0) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 -#define SCH_LOCK(type, _lock) do { \ - if (SCH_READ == (type)) { \ - assert(atomic_load_32(_lock) >= 0); \ - SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) > 0); \ - } else { \ - assert(atomic_load_32(_lock) >= 0); \ - SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY); \ - } \ -} while (0) - -#define SCH_UNLOCK(type, _lock) do { \ - if (SCH_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ - SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRUnLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ - } else { \ - assert(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY); \ - SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWUnLockLatch(_lock); \ - SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ - } \ -} while (0) - - -void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); -void schCleanClusterHb(void* pTrans); -int32_t schLaunchTask(SSchJob *job, SSchTask *task); -int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); +#define SCH_LOCK(type, _lock) \ + do { \ + if (SCH_READ == (type)) { \ + assert(atomic_load_32(_lock) >= 0); \ + SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32(_lock) > 0); \ + } else { \ + assert(atomic_load_32(_lock) >= 0); \ + SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY); \ + } \ + } while (0) + +#define SCH_UNLOCK(type, _lock) \ + do { \ + if (SCH_READ == (type)) { \ + assert(atomic_load_32((_lock)) > 0); \ + SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRUnLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + } else { \ + assert(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY); \ + SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWUnLockLatch(_lock); \ + SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + } \ + } while (0) + +void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); +void schCleanClusterHb(void *pTrans); +int32_t schLaunchTask(SSchJob *job, SSchTask *task); +int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); SSchJob *schAcquireJob(int64_t refId); -int32_t schReleaseJob(int64_t refId); -void schFreeFlowCtrl(SSchJob *pJob); -int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); -int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); -int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough); -int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask); -int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); -int32_t schLaunchFetchTask(SSchJob *pJob); -int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode); -int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction); -int32_t schCloneSMsgSendInfo(void *src, void **dst); -int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob); -void schFreeJobImpl(void *job); -int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx); -int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask); -int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans); -int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code); -void schFreeRpcCtx(SRpcCtx *pCtx); -int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp); -bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus); -int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask); -int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp); -int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp); -void schProcessOnDataFetched(SSchJob *job); -int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask); -void schFreeRpcCtxVal(const void *arg); -int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb); -int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execId); -int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync); -int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus); -int32_t schCancelJob(SSchJob *pJob); -int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode); +int32_t schReleaseJob(int64_t refId); +void schFreeFlowCtrl(SSchJob *pJob); +int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); +int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); +int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough); +int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask); +int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); +int32_t schLaunchFetchTask(SSchJob *pJob); +int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode); +int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction); +int32_t schCloneSMsgSendInfo(void *src, void **dst); +int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob); +void schFreeJobImpl(void *job); +int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx); +int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask); +int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans); +int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code); +void schFreeRpcCtx(SRpcCtx *pCtx); +int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp); +bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus); +int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask); +int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp); +int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp); +void schProcessOnDataFetched(SSchJob *job); +int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask); +void schFreeRpcCtxVal(const void *arg); +int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb); +int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execId); +int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync); +int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus); +int32_t schCancelJob(SSchJob *pJob); +int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode); uint64_t schGenTaskId(void); -void schCloseJobRef(void); -int32_t schAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob); -int32_t schJobFetchRows(SSchJob *pJob); -int32_t schJobFetchRowsA(SSchJob *pJob); -int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); -int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList); -char* schGetOpStr(SCH_OP_TYPE type); -int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); -int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); -int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq); -int32_t schDumpJobExecRes(SSchJob* pJob, SExecResult* pRes); -int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet* pEpSet); -int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32_t rspCode); -void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode); -int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq); -void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode); -int32_t schProcessOnCbBegin(SSchJob** job, SSchTask** task, uint64_t qId, int64_t rId, uint64_t tId); -void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask); -bool schJobDone(SSchJob *pJob); -int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask); -int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask); -int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param); -int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq); -int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode); -int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask); -void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode); -int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry); -int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode); -int32_t schProcessOnJobPartialSuccess(SSchJob *pJob); -void schFreeTask(SSchJob *pJob, SSchTask *pTask); -void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); -int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); -int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); -int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); -int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); -void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode); -int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode); -int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode); -bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync); -int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rspCode); -int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp); +void schCloseJobRef(void); +int32_t schAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob); +int32_t schJobFetchRows(SSchJob *pJob); +int32_t schJobFetchRowsA(SSchJob *pJob); +int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); +int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList); +char *schGetOpStr(SCH_OP_TYPE type); +int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); +int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); +int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq); +int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes); +int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet); +int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode); +void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode); +int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq); +void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode); +int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_t rId, uint64_t tId); +void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask); +bool schJobDone(SSchJob *pJob); +int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask); +int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask); +int32_t schSwitchJobStatus(SSchJob *pJob, int32_t status, void *param); +int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob **job, SCH_OP_TYPE type, SSchedulerReq *pReq); +int32_t schHandleOpEndEvent(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode); +int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask); +void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode); +int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry); +int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode); +int32_t schProcessOnJobPartialSuccess(SSchJob *pJob); +void schFreeTask(SSchJob *pJob, SSchTask *pTask); +void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); +int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); +int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); +int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); +int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); +void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode); +int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode); +int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode); +bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync); +int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rspCode); +int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp); extern SSchDebug gSCHDebug; - #ifdef __cplusplus } #endif diff --git a/source/libs/scheduler/src/schDbg.c b/source/libs/scheduler/src/schDbg.c index a6398522d3358a5867fae171f90a1e5d5fb30077..d6c2b638b8c223dedce533825af4a3c62cf4da3c 100644 --- a/source/libs/scheduler/src/schDbg.c +++ b/source/libs/scheduler/src/schDbg.c @@ -16,19 +16,17 @@ #include "query.h" #include "schInt.h" -tsem_t schdRspSem; +tsem_t schdRspSem; SSchDebug gSCHDebug = {0}; void schdExecCallback(SExecResult* pResult, void* param, int32_t code) { if (code) { pResult->code = code; } - + *(SExecResult*)param = *pResult; taosMemoryFree(pResult); tsem_post(&schdRspSem); } - - diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index c5c2bfb2bb943da6ff85e686cff46b95dc194821..8980d08e891d58f4bbc7e1e88d0d3ea5d1c067ee 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ +#include "catalog.h" +#include "query.h" #include "schInt.h" #include "tmsg.h" -#include "query.h" -#include "catalog.h" #include "tref.h" void schFreeFlowCtrl(SSchJob *pJob) { @@ -25,14 +25,14 @@ void schFreeFlowCtrl(SSchJob *pJob) { } SSchFlowControl *ctrl = NULL; - void *pIter = taosHashIterate(pJob->flowCtrl, NULL); + void *pIter = taosHashIterate(pJob->flowCtrl, NULL); while (pIter) { ctrl = (SSchFlowControl *)pIter; if (ctrl->taskList) { taosArrayDestroy(ctrl->taskList); } - + pIter = taosHashIterate(pJob->flowCtrl, pIter); } @@ -59,7 +59,8 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { return TSDB_CODE_SUCCESS; } - pJob->flowCtrl = taosHashInit(pJob->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + pJob->flowCtrl = + taosHashInit(pJob->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == pJob->flowCtrl) { SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pJob->taskNum); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -73,17 +74,17 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { } int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) { - SSchLevel *pLevel = pTask->level; + SSchLevel *pLevel = pTask->level; SSchFlowControl *ctrl = NULL; - int32_t code = 0; - SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); - + int32_t code = 0; + SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); + ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp)); if (NULL == ctrl) { SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SCH_LOCK(SCH_WRITE, &ctrl->lock); if (ctrl->execTaskNum <= 0) { SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port); @@ -93,8 +94,8 @@ int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) { --ctrl->execTaskNum; ctrl->tableNumSum -= pTask->plan->execNodeStat.tableNum; - SCH_TASK_DLOG("task quota removed, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", - ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); + SCH_TASK_DLOG("task quota removed, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); _return: @@ -104,11 +105,11 @@ _return: } int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { - SSchLevel *pLevel = pTask->level; - int32_t code = 0; + SSchLevel *pLevel = pTask->level; + int32_t code = 0; SSchFlowControl *ctrl = NULL; - SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); - + SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); + do { ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp)); if (NULL == ctrl) { @@ -119,34 +120,34 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { if (HASH_NODE_EXIST(code)) { continue; } - + SCH_TASK_ELOG("taosHashPut flowCtrl failed, size:%d", (int32_t)sizeof(nctrl)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", - ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, nctrl.tableNumSum, nctrl.execTaskNum); + SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + ep->port, pTask->plan->execNodeStat.tableNum, nctrl.tableNumSum, nctrl.execTaskNum); *enough = true; return TSDB_CODE_SUCCESS; } SCH_LOCK(SCH_WRITE, &ctrl->lock); - + if (0 == ctrl->execTaskNum) { ctrl->tableNumSum = pTask->plan->execNodeStat.tableNum; ++ctrl->execTaskNum; - + *enough = true; break; } - + int32_t sum = pTask->plan->execNodeStat.tableNum + ctrl->tableNumSum; - + if (sum <= schMgmt.cfg.maxNodeTableNum) { ctrl->tableNumSum = sum; ++ctrl->execTaskNum; - + *enough = true; break; } @@ -166,24 +167,25 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { *enough = false; ctrl->sorted = false; - + break; } while (true); _return: - SCH_TASK_DLOG("task quota %s added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", - ((*enough)?"":"NOT"), ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); + SCH_TASK_DLOG("task quota %s added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", + ((*enough) ? "" : "NOT"), ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, + ctrl->execTaskNum); SCH_UNLOCK(SCH_WRITE, &ctrl->lock); - + SCH_RET(code); } -int32_t schTaskTableNumCompare(const void* key1, const void* key2) { +int32_t schTaskTableNumCompare(const void *key1, const void *key2) { SSchTask *pTask1 = *(SSchTask **)key1; SSchTask *pTask2 = *(SSchTask **)key2; - + if (pTask1->plan->execNodeStat.tableNum < pTask2->plan->execNodeStat.tableNum) { return 1; } else if (pTask1->plan->execNodeStat.tableNum > pTask2->plan->execNodeStat.tableNum) { @@ -193,22 +195,21 @@ int32_t schTaskTableNumCompare(const void* key1, const void* key2) { } } - int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { SCH_LOCK(SCH_WRITE, &ctrl->lock); - + if (NULL == ctrl->taskList || taosArrayGetSize(ctrl->taskList) <= 0) { SCH_UNLOCK(SCH_WRITE, &ctrl->lock); return TSDB_CODE_SUCCESS; } - int32_t remainNum = schMgmt.cfg.maxNodeTableNum - ctrl->tableNumSum; - int32_t taskNum = taosArrayGetSize(ctrl->taskList); - int32_t code = 0; + int32_t remainNum = schMgmt.cfg.maxNodeTableNum - ctrl->tableNumSum; + int32_t taskNum = taosArrayGetSize(ctrl->taskList); + int32_t code = 0; SSchTask *pTask = NULL; - + if (taskNum > 1 && !ctrl->sorted) { - taosArraySort(ctrl->taskList, schTaskTableNumCompare); // desc order + taosArraySort(ctrl->taskList, schTaskTableNumCompare); // desc order } for (int32_t i = 0; i < taskNum; ++i) { @@ -216,36 +217,36 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) { - SCH_TASK_DLOG("task NOT to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", - ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); + SCH_TASK_DLOG("task NOT to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); continue; } - + ctrl->tableNumSum += pTask->plan->execNodeStat.tableNum; ++ctrl->execTaskNum; taosArrayRemove(ctrl->taskList, i); - - SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", - ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); - + + SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); + SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask)); - + remainNum -= pTask->plan->execNodeStat.tableNum; if (remainNum <= 0) { - SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d", - ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum); - + SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, ep->port, + ctrl->tableNumSum, ctrl->execTaskNum); + break; } if (i < (taskNum - 1)) { SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList); if (remainNum < pLastTask->plan->execNodeStat.tableNum) { - SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d, smallestInList:%d", - ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum); - + SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d, smallestInList:%d", + ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum); + break; } } @@ -253,7 +254,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { --i; --taskNum; } - + _return: SCH_UNLOCK(SCH_WRITE, &ctrl->lock); @@ -261,11 +262,10 @@ _return: if (code) { code = schProcessOnTaskFailure(pJob, pTask, code); } - + SCH_RET(code); } - int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) { if (!SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { return TSDB_CODE_SUCCESS; @@ -274,17 +274,16 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schDecTaskFlowQuota(pJob, pTask)); SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); - + SSchFlowControl *ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp)); if (NULL == ctrl) { SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - - int32_t code = schLaunchTasksInFlowCtrlListImpl(pJob, ctrl);; - SCH_ERR_RET(code); - - return code; // to avoid compiler error -} + int32_t code = schLaunchTasksInFlowCtrlListImpl(pJob, ctrl); + ; + SCH_ERR_RET(code); + return code; // to avoid compiler error +} diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 69495c8b7aa0901807ed81a146418f0c6b6264db..bb8451ba37bf9dbc79e762f421cde624b440b9f7 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -52,9 +52,8 @@ _return: bool schJobDone(SSchJob *pJob) { int8_t status = SCH_GET_JOB_STATUS(pJob); - - return (status == JOB_TASK_STATUS_FAIL || status == JOB_TASK_STATUS_DROP || - status == JOB_TASK_STATUS_SUCC); + + return (status == JOB_TASK_STATUS_FAIL || status == JOB_TASK_STATUS_DROP || status == JOB_TASK_STATUS_SUCC); } FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { @@ -221,7 +220,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("parents info, the %d parent TID 0x%" PRIx64, n, (*parentTask)->taskId); + SCH_TASK_DLOG("parents info, the %d parent TID 0x%" PRIx64, n, (*parentTask)->taskId); } SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum); @@ -235,7 +234,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - SSchTask* pTask = taosArrayGet(pLevel->subTasks, 0); + SSchTask *pTask = taosArrayGet(pLevel->subTasks, 0); if (SUBPLAN_TYPE_MODIFY != pTask->plan->subplanType) { pJob->attr.needFetch = true; } @@ -244,7 +243,6 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { return TSDB_CODE_SUCCESS; } - int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) { if (!SCH_IS_DATA_BIND_QRY_TASK(pTask)) { return TSDB_CODE_SUCCESS; @@ -255,7 +253,6 @@ int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { int32_t code = 0; pJob->queryId = pDag->queryId; @@ -365,7 +362,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_JRET(schBuildTaskRalation(pJob, planToTask)); _return: - + if (planToTask) { taosHashCleanup(planToTask); } @@ -373,8 +370,7 @@ _return: SCH_RET(code); } - -int32_t schDumpJobExecRes(SSchJob* pJob, SExecResult* pRes) { +int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { pRes->code = atomic_load_32(&pJob->errCode); pRes->numOfRows = pJob->resNumOfRows; pRes->res = pJob->execRes.res; @@ -387,13 +383,13 @@ int32_t schDumpJobExecRes(SSchJob* pJob, SExecResult* pRes) { return TSDB_CODE_SUCCESS; } -int32_t schDumpJobFetchRes(SSchJob* pJob, void** pData) { +int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { int32_t code = 0; - + SCH_LOCK(SCH_WRITE, &pJob->resLock); pJob->fetched = true; - + if (pJob->fetchRes && ((SRetrieveTableRsp *)pJob->fetchRes)->completed) { SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_SUCC, NULL)); } @@ -422,12 +418,12 @@ int32_t schDumpJobFetchRes(SSchJob* pJob, void** pData) { _return: SCH_UNLOCK(SCH_WRITE, &pJob->resLock); - + return code; } -int32_t schNotifyUserExecRes(SSchJob* pJob) { - SExecResult* pRes = taosMemoryCalloc(1, sizeof(SExecResult)); +int32_t schNotifyUserExecRes(SSchJob *pJob) { + SExecResult *pRes = taosMemoryCalloc(1, sizeof(SExecResult)); if (pRes) { schDumpJobExecRes(pJob, pRes); } @@ -439,9 +435,9 @@ int32_t schNotifyUserExecRes(SSchJob* pJob) { return TSDB_CODE_SUCCESS; } -int32_t schNotifyUserFetchRes(SSchJob* pJob) { - void* pRes = NULL; - +int32_t schNotifyUserFetchRes(SSchJob *pJob) { + void *pRes = NULL; + schDumpJobFetchRes(pJob, &pRes); SCH_JOB_DLOG("sch start to invoke fetch cb, code: %s", tstrerror(pJob->errCode)); @@ -453,17 +449,17 @@ int32_t schNotifyUserFetchRes(SSchJob* pJob) { void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) { SCH_LOCK(SCH_WRITE, &pJob->opStatus.lock); - + if (SCH_OP_NULL == pJob->opStatus.op) { SCH_JOB_DLOG("job not in any operation, no need to post job res, status:%s", jobTaskStatusStr(pJob->status)); goto _return; } - + if (op && pJob->opStatus.op != op) { SCH_JOB_ELOG("job in operation %s mis-match with expected %s", schGetOpStr(pJob->opStatus.op), schGetOpStr(op)); goto _return; } - + if (SCH_JOB_IN_SYNC_OP(pJob)) { SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); tsem_post(&pJob->rspSem); @@ -487,7 +483,7 @@ _return: int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { schUpdateJobErrCode(pJob, errCode); - + int32_t code = atomic_load_32(&pJob->errCode); if (code) { SCH_JOB_DLOG("job failed with error %s", tstrerror(code)); @@ -507,9 +503,7 @@ int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } -int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) { - SCH_RET(schProcessOnJobFailure(pJob, errCode)); -} +int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) { SCH_RET(schProcessOnJobFailure(pJob, errCode)); } int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) { if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) { @@ -520,8 +514,7 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - -int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { +int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { if (schChkCurrentOp(pJob, SCH_OP_FETCH, -1)) { SCH_ERR_RET(schLaunchFetchTask(pJob)); } else { @@ -531,9 +524,7 @@ int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } -void schProcessOnDataFetched(SSchJob *pJob) { - schPostJobRes(pJob, SCH_OP_FETCH); -} +void schProcessOnDataFetched(SSchJob *pJob) { schPostJobRes(pJob, SCH_OP_FETCH); } int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); @@ -548,14 +539,13 @@ int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRs return TSDB_CODE_SUCCESS; } - int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { if (!SCH_IS_QUERY_JOB(pJob)) { return TSDB_CODE_SUCCESS; } SSchLevel *pLevel = pTask->level; - int32_t doneNum = atomic_add_fetch_32(&pLevel->taskDoneNum, 1); + int32_t doneNum = atomic_add_fetch_32(&pLevel->taskDoneNum, 1); if (doneNum == pLevel->taskNum) { pJob->levelIdx--; @@ -566,7 +556,7 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { if (pTask->children && taosArrayGetSize(pTask->children) > 0) { continue; } - + SCH_ERR_RET(schLaunchTask(pJob, pTask)); } } @@ -577,11 +567,11 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp) { if (rsp->tbFName[0]) { SCH_LOCK(SCH_WRITE, &pJob->resLock); - + if (NULL == pJob->execRes.res) { pJob->execRes.res = taosArrayInit(pJob->taskNum, sizeof(STbVerInfo)); if (NULL == pJob->execRes.res) { - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -610,7 +600,6 @@ int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask) { return TSDB_CODE_SUCCESS; } - int32_t schLaunchJob(SSchJob *pJob) { if (EXPLAIN_MODE_STATIC == pJob->attr.explainMode) { SCH_ERR_RET(qExecStaticExplain(pJob->pDag, (SRetrieveTableRsp **)&pJob->fetchRes)); @@ -623,11 +612,10 @@ int32_t schLaunchJob(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } - void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->execTasks); -// schDropTaskInHashList(pJob, pJob->succTasks); -// schDropTaskInHashList(pJob, pJob->failTasks); + // schDropTaskInHashList(pJob, pJob->succTasks); + // schDropTaskInHashList(pJob, pJob->failTasks); } void schFreeJobImpl(void *job) { @@ -659,10 +647,10 @@ void schFreeJobImpl(void *job) { schFreeFlowCtrl(pJob); taosHashCleanup(pJob->execTasks); -// taosHashCleanup(pJob->failTasks); -// taosHashCleanup(pJob->succTasks); + // taosHashCleanup(pJob->failTasks); + // taosHashCleanup(pJob->succTasks); taosHashCleanup(pJob->taskList); - + taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); taosArrayDestroy(pJob->dataSrcTasks); @@ -688,19 +676,19 @@ void schFreeJobImpl(void *job) { } int32_t schJobFetchRows(SSchJob *pJob) { - int32_t code = 0; + int32_t code = 0; if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC)) { SCH_ERR_RET(schLaunchFetchTask(pJob)); - + if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); tsem_wait(&pJob->rspSem); - SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); + SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); } } else { if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { - SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); + SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); } else { schPostJobRes(pJob, SCH_OP_FETCH); } @@ -736,9 +724,9 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { } else { pJob->nodeList = taosArrayDup(pReq->pNodeList); } - - pJob->taskList = - taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + + pJob->taskList = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, + HASH_ENTRY_LOCK); if (NULL == pJob->taskList) { SCH_JOB_ELOG("taosHashInit %d taskList failed", pReq->pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -750,8 +738,8 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { SCH_ERR_JRET(qExecExplainBegin(pReq->pDag, &pJob->explainCtx, pReq->startTs)); } - pJob->execTasks = - taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->execTasks = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, + HASH_ENTRY_LOCK); if (NULL == pJob->execTasks) { SCH_JOB_ELOG("taosHashInit %d execTasks failed", pReq->pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -769,7 +757,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { *pJobId = pJob->refId; - SCH_JOB_DLOG("job refId:0x%" PRIx64" created", pJob->refId); + SCH_JOB_DLOG("job refId:0x%" PRIx64 " created", pJob->refId); return TSDB_CODE_SUCCESS; @@ -782,31 +770,31 @@ _return: } else { taosRemoveRef(schMgmt.jobRef, pJob->refId); } - + SCH_RET(code); } int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) { int32_t code = 0; - qDebug("QID:0x%" PRIx64 " sch job refId 0x%"PRIx64 " started", pReq->pDag->queryId, pJob->refId); + qDebug("QID:0x%" PRIx64 " sch job refId 0x%" PRIx64 " started", pReq->pDag->queryId, pJob->refId); SCH_ERR_RET(schLaunchJob(pJob)); - + if (pReq->syncReq) { SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); tsem_wait(&pJob->rspSem); } SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); - + return TSDB_CODE_SUCCESS; } -void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode) { +void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode) { if (NULL == pReq || pReq->syncReq) { return; } - + if (pReq->execFp) { (*pReq->execFp)(NULL, pReq->cbParam, errCode); } else if (pReq->fetchFp) { @@ -827,16 +815,17 @@ bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync) { return r; } -void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode) { +void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode) { int32_t op = 0; - + switch (type) { case SCH_OP_EXEC: if (pReq && pReq->syncReq) { SCH_LOCK(SCH_WRITE, &pJob->opStatus.lock); op = atomic_val_compare_exchange_32(&pJob->opStatus.op, type, SCH_OP_NULL); if (SCH_OP_NULL == op || op != type) { - SCH_JOB_ELOG("job not in %s operation, op:%s, status:%s", schGetOpStr(type), schGetOpStr(op), jobTaskStatusStr(pJob->status)); + SCH_JOB_ELOG("job not in %s operation, op:%s, status:%s", schGetOpStr(type), schGetOpStr(op), + jobTaskStatusStr(pJob->status)); } SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); schDumpJobExecRes(pJob, pReq->pExecRes); @@ -847,7 +836,8 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int SCH_LOCK(SCH_WRITE, &pJob->opStatus.lock); op = atomic_val_compare_exchange_32(&pJob->opStatus.op, type, SCH_OP_NULL); if (SCH_OP_NULL == op || op != type) { - SCH_JOB_ELOG("job not in %s operation, op:%s, status:%s", schGetOpStr(type), schGetOpStr(op), jobTaskStatusStr(pJob->status)); + SCH_JOB_ELOG("job not in %s operation, op:%s, status:%s", schGetOpStr(type), schGetOpStr(op), + jobTaskStatusStr(pJob->status)); } SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); } @@ -866,10 +856,10 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int SCH_JOB_DLOG("job end %s operation with code %s", schGetOpStr(type), tstrerror(errCode)); } -int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq) { +int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq) { int32_t code = 0; - int8_t status = SCH_GET_JOB_STATUS(pJob); - + int8_t status = SCH_GET_JOB_STATUS(pJob); + switch (type) { case SCH_OP_EXEC: SCH_LOCK(SCH_WRITE, &pJob->opStatus.lock); @@ -879,9 +869,9 @@ int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } - + SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); - + pJob->opStatus.syncReq = pReq->syncReq; SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); break; @@ -893,16 +883,16 @@ int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); } - + SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); - + pJob->userRes.fetchRes = pReq->pFetchRes; pJob->userRes.fetchFp = pReq->fetchFp; pJob->userRes.cbParam = pReq->cbParam; - + pJob->opStatus.syncReq = pReq->syncReq; SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - + if (!SCH_JOB_NEED_FETCH(pJob)) { SCH_JOB_ELOG("no need to fetch data, status:%s", SCH_GET_JOB_STATUS_STR(pJob)); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -912,7 +902,7 @@ int32_t schProcessOnOpBegin(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + break; case SCH_OP_GET_STATUS: if (pJob->status < JOB_TASK_STATUS_INIT || pJob->levelNum <= 0 || NULL == pJob->levels) { @@ -941,23 +931,23 @@ void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { if (errCode) { schHandleJobFailure(pJob, errCode); } - + if (pJob) { schReleaseJob(pJob->refId); } } -int32_t schProcessOnCbBegin(SSchJob** job, SSchTask** task, uint64_t qId, int64_t rId, uint64_t tId) { +int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_t rId, uint64_t tId) { int32_t code = 0; - int8_t status = 0; + int8_t status = 0; SSchTask *pTask = NULL; - SSchJob *pJob = schAcquireJob(rId); + SSchJob *pJob = schAcquireJob(rId); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId); SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST); } - + if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("will not do further processing cause of job status %s", jobTaskStatusStr(status)); SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); @@ -980,9 +970,6 @@ _return: if (pJob) { schReleaseJob(rId); } - + SCH_RET(code); } - - - diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index 64cda573f1fa8bd022a548787151b2b9fd64d44f..66d58c95d1c3a36938fb0b5b1fa420c558a33e4a 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -29,25 +29,25 @@ int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) { case JOB_TASK_STATUS_INIT: break; case JOB_TASK_STATUS_EXEC: - SCH_ERR_JRET(schExecJob(pJob, (SSchedulerReq*)param)); + SCH_ERR_JRET(schExecJob(pJob, (SSchedulerReq*)param)); break; case JOB_TASK_STATUS_PART_SUCC: SCH_ERR_JRET(schProcessOnJobPartialSuccess(pJob)); break; case JOB_TASK_STATUS_SUCC: break; - case JOB_TASK_STATUS_FAIL: + case JOB_TASK_STATUS_FAIL: SCH_RET(schProcessOnJobFailure(pJob, (param ? *(int32_t*)param : 0))); break; case JOB_TASK_STATUS_DROP: schProcessOnJobDropped(pJob, *(int32_t*)param); - + if (taosRemoveRef(schMgmt.jobRef, pJob->refId)) { SCH_JOB_ELOG("remove job from job list failed, refId:0x%" PRIx64, pJob->refId); } else { SCH_JOB_DLOG("job removed from jobRef list, refId:0x%" PRIx64, pJob->refId); } - break; + break; default: { SCH_JOB_ELOG("unknown job status %d", status); SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); @@ -62,7 +62,7 @@ _return: } int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { - SSchJob *pJob = schAcquireJob(jobId); + SSchJob* pJob = schAcquireJob(jobId); if (NULL == pJob) { qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); @@ -75,12 +75,12 @@ int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SS int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int32_t errCode) { int32_t code = errCode; - + if (NULL == pJob) { schDirectPostJobRes(pReq, errCode); SCH_RET(code); } - + schProcessOnOpEnd(pJob, type, pReq, errCode); if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) { @@ -91,5 +91,3 @@ int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq return code; } - - diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index b6f96a188e4fc06f365b6ccf527c3587abb959af..33bf50d85313a6fa5f9709568f525bd98894c215 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -16,12 +16,12 @@ #include "catalog.h" #include "command.h" #include "query.h" +#include "qworker.h" #include "schInt.h" +#include "tglobal.h" #include "tmsg.h" #include "tref.h" #include "trpc.h" -#include "qworker.h" -#include "tglobal.h" void schFreeTask(SSchJob *pJob, SSchTask *pTask) { schDeregisterTaskHb(pJob, pTask); @@ -94,7 +94,7 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_LOCAL_EXEC_TASK(pJob, pTask)) { return TSDB_CODE_SUCCESS; } - + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); if (NULL == addr) { SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", pTask->candidateIdx, @@ -162,14 +162,15 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v if (dropExecNode) { SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId)); } - + schUpdateTaskExecNode(pJob, pTask, handle, execId); if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it - SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); + SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, + pTask->execId, pTask->waitRetry); SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); } - + SCH_SET_TASK_HANDLE(pTask, handle); return TSDB_CODE_SUCCESS; @@ -837,17 +838,18 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { goto _return; } - SSchTask *pTask = NULL; - SSchJob *pJob = NULL; + SSchTask *pTask = NULL; + SSchJob *pJob = NULL; for (int32_t i = 0; i < resNum; ++i) { - SExplainLocalRsp* localRsp = taosArrayGet(pExplainRes, i); + SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); pJob = schAcquireJob(localRsp->rId); if (NULL == pJob) { - qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, + localRsp->tId, localRsp->rId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_NOT_EXIST); } @@ -857,16 +859,17 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { schReleaseJob(pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } - + code = schGetTaskInJob(pJob, localRsp->tId, &pTask); if (TSDB_CODE_SUCCESS == code) { code = schProcessExplainRsp(pJob, pTask, &localRsp->rsp); } - + schReleaseJob(pJob->refId); - qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, + localRsp->tId, code); SCH_ERR_JRET(code); @@ -879,7 +882,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) { _return: for (int32_t i = 0; i < resNum; ++i) { - SExplainLocalRsp* localRsp = taosArrayGet(pExplainRes, i); + SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i); tFreeSExplainRsp(&localRsp->rsp); } @@ -890,7 +893,7 @@ _return: int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { SSubplan *plan = pTask->plan; - int32_t code = 0; + int32_t code = 0; if (NULL == pTask->msg) { // TODO add more detailed reason for failure code = qSubPlanToMsg(plan, &pTask->msg, &pTask->msgLen); @@ -899,7 +902,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { pTask->msgLen); SCH_ERR_RET(code); } else if (tsQueryPlannerTrace) { - char *msg = NULL; + char *msg = NULL; int32_t msgLen = 0; qSubPlanToString(plan, &msg, &msgLen); SCH_TASK_DLOGL("physical plan len:%d, %s", msgLen, msg); @@ -910,20 +913,20 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask)); if (SCH_IS_QUERY_JOB(pJob)) { - SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); +// SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); } - + SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); } int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { - //SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); + // SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); if (NULL == schMgmt.queryMgmt) { SCH_ERR_RET(qWorkerInit(NODE_TYPE_CLIENT, CLIENT_HANDLE, (void **)&schMgmt.queryMgmt, NULL)); } SArray *explainRes = NULL; - SQWMsg qwMsg = {0}; + SQWMsg qwMsg = {0}; qwMsg.msgInfo.taskType = TASK_TYPE_TEMP; qwMsg.msgInfo.explain = SCH_IS_EXPLAIN_JOB(pJob); qwMsg.msgInfo.needFetch = SCH_TASK_NEED_FETCH(pTask); @@ -934,8 +937,9 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); } - - SCH_ERR_RET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &qwMsg, explainRes)); + + SCH_ERR_RET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, + pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); @@ -958,17 +962,17 @@ int32_t schLaunchTaskImpl(void *param) { if (pCtx->asyncLaunch) { SCH_LOCK_TASK(pTask); } - - int8_t status = 0; - int32_t code = 0; + + int8_t status = 0; + int32_t code = 0; atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; pTask->retryTimes++; pTask->waitRetry = false; - SCH_TASK_DLOG("start to launch %s task, execId %d, retry %d", SCH_IS_LOCAL_EXEC_TASK(pJob, pTask) ? "LOCAL" : "REMOTE", - pTask->execId, pTask->retryTimes); + SCH_TASK_DLOG("start to launch %s task, execId %d, retry %d", + SCH_IS_LOCAL_EXEC_TASK(pJob, pTask) ? "LOCAL" : "REMOTE", pTask->execId, pTask->retryTimes); SCH_LOG_TASK_START_TS(pTask); @@ -989,6 +993,12 @@ int32_t schLaunchTaskImpl(void *param) { SCH_ERR_JRET(schLaunchRemoteTask(pJob, pTask)); } +#if 0 + if (SCH_IS_QUERY_JOB(pJob)) { + SCH_ERR_JRET(schEnsureHbConnection(pJob, pTask)); + } +#endif + _return: if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { @@ -1086,19 +1096,20 @@ int32_t schExecRemoteFetch(SSchJob *pJob, SSchTask *pTask) { } int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { - void *pRsp = NULL; + void *pRsp = NULL; SArray *explainRes = NULL; if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); } - SCH_ERR_RET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &pRsp, explainRes)); + SCH_ERR_RET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, + pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); } - + SCH_ERR_RET(schProcessFetchRsp(pJob, pTask, pRsp, TSDB_CODE_SUCCESS)); return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 6f12780ff932a01a3303a175ec0d2be45b0875a2..7099dbb2c0c20682ee59bb4b3b0275c2684dedca 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -21,21 +21,21 @@ #include "tref.h" #include "trpc.h" -FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { - qDebug("sch acquire jobId:0x%"PRIx64, refId); - return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); +FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { + qDebug("sch acquire jobId:0x%" PRIx64, refId); + return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } -FORCE_INLINE int32_t schReleaseJob(int64_t refId) { +FORCE_INLINE int32_t schReleaseJob(int64_t refId) { if (0 == refId) { return TSDB_CODE_SUCCESS; } - - qDebug("sch release jobId:0x%"PRIx64, refId); - return taosReleaseRef(schMgmt.jobRef, refId); + + qDebug("sch release jobId:0x%" PRIx64, refId); + return taosReleaseRef(schMgmt.jobRef, refId); } -char* schGetOpStr(SCH_OP_TYPE type) { +char *schGetOpStr(SCH_OP_TYPE type) { switch (type) { case SCH_OP_NULL: return "NULL"; @@ -53,28 +53,28 @@ char* schGetOpStr(SCH_OP_TYPE type) { void schFreeHbTrans(SSchHbTrans *pTrans) { rpcReleaseHandle(pTrans->trans.pHandle, TAOS_CONN_CLIENT); - schFreeRpcCtx(&pTrans->rpcCtx); + schFreeRpcCtx(&pTrans->rpcCtx); } -void schCleanClusterHb(void* pTrans) { +void schCleanClusterHb(void *pTrans) { SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); SSchHbTrans *hb = taosHashIterate(schMgmt.hbConnections, NULL); while (hb) { if (hb->trans.pTrans == pTrans) { - SQueryNodeEpId* pEpId = taosHashGetKey(hb, NULL); + SQueryNodeEpId *pEpId = taosHashGetKey(hb, NULL); schFreeHbTrans(hb); taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); } - + hb = taosHashIterate(schMgmt.hbConnections, hb); } - + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); } int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId) { - int32_t code = 0; + int32_t code = 0; SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); @@ -94,7 +94,6 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep return TSDB_CODE_SUCCESS; } - int32_t schAddHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { int32_t code = 0; SSchHbTrans hb = {0}; @@ -155,13 +154,13 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { if (!pTask->registerdHb) { return; } - + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; - SEp* pEp = SCH_GET_CUR_EP(addr); + SEp *pEp = SCH_GET_CUR_EP(addr); strcpy(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; @@ -180,24 +179,22 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { } else { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); } - + pTask->registerdHb = false; } - - int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; - SEp* pEp = SCH_GET_CUR_EP(addr); + SEp *pEp = SCH_GET_CUR_EP(addr); strcpy(epId.ep.fqdn, pEp->fqdn); epId.ep.port = pEp->port; SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId)); - + pTask->registerdHb = true; return TSDB_CODE_SUCCESS; @@ -226,7 +223,6 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { return TSDB_CODE_SUCCESS; } - void schCloseJobRef(void) { if (!atomic_load_8((int8_t *)&schMgmt.exit)) { return; @@ -242,7 +238,7 @@ uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); } uint64_t schGenUUID(void) { static uint64_t hashId = 0; - static int32_t requestSerialId = 0; + static int32_t requestSerialId = 0; if (hashId == 0) { char uid[64] = {0}; @@ -254,15 +250,14 @@ uint64_t schGenUUID(void) { } } - int64_t ts = taosGetTimestampMs(); - uint64_t pid = taosGetPId(); - int32_t val = atomic_add_fetch_32(&requestSerialId, 1); + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&requestSerialId, 1); uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; } - void schFreeRpcCtxVal(const void *arg) { if (NULL == arg) { return; @@ -307,5 +302,3 @@ int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTas return TSDB_CODE_SUCCESS; } - - diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 0ccaef385744738969973b19a38182cbac4399d9..c4b45649a43691c737dd2b929f763d9c704dcd9d 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -14,10 +14,10 @@ */ #include "query.h" +#include "qworker.h" #include "schInt.h" #include "tmsg.h" #include "tref.h" -#include "qworker.h" SSchedulerMgmt schMgmt = { .jobRef = -1, @@ -35,7 +35,7 @@ int32_t schedulerInit() { schMgmt.cfg.enableReSchedule = true; qDebug("schedule policy init to %d", schMgmt.cfg.schPolicy); - + schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); if (schMgmt.jobRef < 0) { qError("init schduler jobRef failed, num:%u", schMgmt.cfg.maxJobNum); @@ -61,7 +61,7 @@ int32_t schedulerInit() { int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJobId) { qDebug("scheduler %s exec job start", pReq->syncReq ? "SYNC" : "ASYNC"); - int32_t code = 0; + int32_t code = 0; SSchJob *pJob = NULL; SCH_ERR_JRET(schInitJob(pJobId, pReq)); @@ -73,7 +73,7 @@ int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJobId) { SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_EXEC, pReq)); _return: - + SCH_RET(schHandleOpEndEvent(pJob, SCH_OP_EXEC, pReq, code)); } @@ -144,7 +144,7 @@ int32_t schedulerEnableReSchedule(bool enableResche) { return TSDB_CODE_SUCCESS; } -void schedulerFreeJob(int64_t* jobId, int32_t errCode) { +void schedulerFreeJob(int64_t *jobId, int32_t errCode) { if (0 == *jobId) { return; } @@ -155,10 +155,9 @@ void schedulerFreeJob(int64_t* jobId, int32_t errCode) { return; } - SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", errCode:0x%x", *jobId, errCode); - + SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode)); schHandleJobDrop(pJob, errCode); - + schReleaseJob(*jobId); *jobId = 0; } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index ca2122ed8f433c4b8dd70eb120e0eba972dda9fe..97e14b617c1304f7b3003e2f77e9bfee79f05d0b 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -30,15 +30,12 @@ #endif #include "os.h" -#include "tglobal.h" -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" #include "catalog.h" #include "scheduler.h" #include "taos.h" #include "tdatablock.h" #include "tdef.h" +#include "tglobal.h" #include "trpc.h" #include "tvariant.h" @@ -56,8 +53,9 @@ namespace { -extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); -extern "C" int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode); +extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, + int32_t rspCode); +extern "C" int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode); int64_t insertJobRefId = 0; int64_t queryJobRefId = 0; @@ -66,16 +64,15 @@ uint64_t schtMergeTemplateId = 0x4; uint64_t schtFetchTaskId = 0; uint64_t schtQueryId = 1; -bool schtTestStop = false; -bool schtTestDeadLoop = false; +bool schtTestStop = false; +bool schtTestDeadLoop = false; int32_t schtTestMTRunSec = 10; int32_t schtTestPrintNum = 1000; int32_t schtStartFetch = 0; - void schtInitLogFile() { - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; tsAsyncLog = 0; qDebugFlag = 159; @@ -84,24 +81,22 @@ void schtInitLogFile() { if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } - } -void schtQueryCb(SExecResult* pResult, void* param, int32_t code) { +void schtQueryCb(SExecResult *pResult, void *param, int32_t code) { assert(TSDB_CODE_SUCCESS == code); - *(int32_t*)param = 1; + *(int32_t *)param = 1; } - void schtBuildQueryDag(SQueryPlan *dag) { uint64_t qId = schtQueryId; - + dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); - SNodeListNode *scan = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SNodeListNode *merge = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - + SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); @@ -117,7 +112,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); - scanPlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); + scanPlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan->msgType = TDMT_SCH_QUERY; mergePlan->id.queryId = qId; @@ -129,32 +124,32 @@ void schtBuildQueryDag(SQueryPlan *dag) { mergePlan->pChildren = nodesMakeList(); mergePlan->pParents = NULL; - mergePlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); + mergePlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->msgType = TDMT_SCH_QUERY; merge->pNodeList = nodesMakeList(); scan->pNodeList = nodesMakeList(); - nodesListAppend(merge->pNodeList, (SNode*)mergePlan); - nodesListAppend(scan->pNodeList, (SNode*)scanPlan); + nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + nodesListAppend(scan->pNodeList, (SNode *)scanPlan); - nodesListAppend(mergePlan->pChildren, (SNode*)scanPlan); - nodesListAppend(scanPlan->pParents, (SNode*)mergePlan); + nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); + nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); - nodesListAppend(dag->pSubplans, (SNode*)merge); - nodesListAppend(dag->pSubplans, (SNode*)scan); + nodesListAppend(dag->pSubplans, (SNode *)merge); + nodesListAppend(dag->pSubplans, (SNode *)scan); } void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { uint64_t qId = schtQueryId; - int32_t scanPlanNum = 20; - + int32_t scanPlanNum = 20; + dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); - SNodeListNode *scan = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SNodeListNode *merge = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - + SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(scanPlanNum, sizeof(SSubplan)); SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); @@ -180,13 +175,13 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { scanPlan[i].pChildren = NULL; scanPlan[i].level = 1; scanPlan[i].pParents = nodesMakeList(); - scanPlan[i].pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); + scanPlan[i].pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan[i].msgType = TDMT_SCH_QUERY; - nodesListAppend(scanPlan[i].pParents, (SNode*)mergePlan); - nodesListAppend(mergePlan->pChildren, (SNode*)(scanPlan + i)); + nodesListAppend(scanPlan[i].pParents, (SNode *)mergePlan); + nodesListAppend(mergePlan->pChildren, (SNode *)(scanPlan + i)); - nodesListAppend(scan->pNodeList, (SNode*)(scanPlan + i)); + nodesListAppend(scan->pNodeList, (SNode *)(scanPlan + i)); } mergePlan->id.queryId = qId; @@ -197,29 +192,25 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pParents = NULL; - mergePlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); + mergePlan->pNode = (SPhysiNode *)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->msgType = TDMT_SCH_QUERY; - nodesListAppend(merge->pNodeList, (SNode*)mergePlan); - - nodesListAppend(dag->pSubplans, (SNode*)merge); - nodesListAppend(dag->pSubplans, (SNode*)scan); -} - - -void schtFreeQueryDag(SQueryPlan *dag) { + nodesListAppend(merge->pNodeList, (SNode *)mergePlan); + nodesListAppend(dag->pSubplans, (SNode *)merge); + nodesListAppend(dag->pSubplans, (SNode *)scan); } +void schtFreeQueryDag(SQueryPlan *dag) {} void schtBuildInsertDag(SQueryPlan *dag) { uint64_t qId = 0x0000000000000002; - + dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = nodesMakeList(); - SNodeListNode *inserta = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - + SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST); + SSubplan *insertPlan = (SSubplan *)taosMemoryCalloc(2, sizeof(SSubplan)); insertPlan[0].id.queryId = qId; @@ -235,7 +226,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[0].pChildren = NULL; insertPlan[0].pParents = NULL; insertPlan[0].pNode = NULL; - insertPlan[0].pDataSink = (SDataSinkNode*)taosMemoryCalloc(1, sizeof(SDataSinkNode)); + insertPlan[0].pDataSink = (SDataSinkNode *)taosMemoryCalloc(1, sizeof(SDataSinkNode)); insertPlan[0].msgType = TDMT_VND_SUBMIT; insertPlan[1].id.queryId = qId; @@ -251,48 +242,43 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[1].pChildren = NULL; insertPlan[1].pParents = NULL; insertPlan[1].pNode = NULL; - insertPlan[1].pDataSink = (SDataSinkNode*)taosMemoryCalloc(1, sizeof(SDataSinkNode)); + insertPlan[1].pDataSink = (SDataSinkNode *)taosMemoryCalloc(1, sizeof(SDataSinkNode)); insertPlan[1].msgType = TDMT_VND_SUBMIT; inserta->pNodeList = nodesMakeList(); - nodesListAppend(inserta->pNodeList, (SNode*)insertPlan); + nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); insertPlan += 1; - nodesListAppend(inserta->pNodeList, (SNode*)insertPlan); + nodesListAppend(inserta->pNodeList, (SNode *)insertPlan); - nodesListAppend(dag->pSubplans, (SNode*)inserta); + nodesListAppend(dag->pSubplans, (SNode *)inserta); } - -int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { +int32_t schtPlanToString(const SSubplan *subplan, char **str, int32_t *len) { *str = (char *)taosMemoryCalloc(1, 20); *len = 20; return 0; } -void schtExecNode(SSubplan* subplan, uint64_t groupId, SQueryNodeAddr* ep) { +void schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) {} -} - -void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { - -} +void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {} void schtSetPlanToString() { static Stub stub; stub.set(qSubPlanToString, schtPlanToString); { #ifdef WINDOWS - AddrAny any; - std::map result; + AddrAny any; + std::map result; any.get_func_addr("qSubPlanToString", result); #endif #ifdef LINUX - AddrAny any("libplanner.so"); - std::map result; + AddrAny any("libplanner.so"); + std::map result; any.get_global_func_addr_dynsym("^qSubPlanToString$", result); #endif - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, schtPlanToString); } } @@ -303,16 +289,16 @@ void schtSetExecNode() { stub.set(qSetSubplanExecutionNode, schtExecNode); { #ifdef WINDOWS - AddrAny any; - std::map result; + AddrAny any; + std::map result; any.get_func_addr("qSetSubplanExecutionNode", result); #endif #ifdef LINUX - AddrAny any("libplanner.so"); - std::map result; + AddrAny any("libplanner.so"); + std::map result; any.get_global_func_addr_dynsym("^qSetSubplanExecutionNode$", result); #endif - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, schtExecNode); } } @@ -323,22 +309,22 @@ void schtSetRpcSendRequest() { stub.set(rpcSendRequest, schtRpcSendRequest); { #ifdef WINDOWS - AddrAny any; - std::map result; + AddrAny any; + std::map result; any.get_func_addr("rpcSendRequest", result); #endif #ifdef LINUX - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRequest$", result); #endif - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, schtRpcSendRequest); } } } -int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo) { +int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet *epSet, int64_t *pTransporterId, SMsgSendInfo *pInfo) { if (pInfo) { taosMemoryFreeClear(pInfo->param); taosMemoryFreeClear(pInfo->msgInfo.pData); @@ -347,32 +333,30 @@ int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTr return 0; } - void schtSetAsyncSendMsgToServer() { static Stub stub; stub.set(asyncSendMsgToServer, schtAsyncSendMsgToServer); { #ifdef WINDOWS - AddrAny any; - std::map result; + AddrAny any; + std::map result; any.get_func_addr("asyncSendMsgToServer", result); #endif #ifdef LINUX - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^asyncSendMsgToServer$", result); #endif - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, schtAsyncSendMsgToServer); } } } - void *schtSendRsp(void *param) { SSchJob *pJob = NULL; - int64_t job = 0; - int32_t code = 0; + int64_t job = 0; + int32_t code = 0; while (true) { job = *(int64_t *)param; @@ -384,7 +368,7 @@ void *schtSendRsp(void *param) { } pJob = schAcquireJob(job); - + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; @@ -392,9 +376,9 @@ void *schtSendRsp(void *param) { SSubmitRsp rsp = {0}; rsp.affectedRows = 10; schHandleResponseMsg(pJob, task, TDMT_VND_SUBMIT_RSP, (char *)&rsp, sizeof(rsp), 0); - + pIter = taosHashIterate(pJob->execTasks, pIter); - } + } schReleaseJob(job); @@ -402,28 +386,27 @@ void *schtSendRsp(void *param) { } void *schtCreateFetchRspThread(void *param) { - int64_t job = *(int64_t *)param; - SSchJob* pJob = schAcquireJob(job); + int64_t job = *(int64_t *)param; + SSchJob *pJob = schAcquireJob(job); taosSsleep(1); - int32_t code = 0; + int32_t code = 0; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; - + code = schHandleResponseMsg(pJob, pJob->fetchTask, TDMT_SCH_FETCH_RSP, (char *)rsp, sizeof(*rsp), 0); schReleaseJob(job); - + assert(code == 0); return NULL; } - void *schtFetchRspThread(void *aa) { - SDataBuf dataBuf = {0}; - SSchTaskCallbackParam* param = NULL; + SDataBuf dataBuf = {0}; + SSchTaskCallbackParam *param = NULL; while (!schtTestStop) { if (0 == atomic_val_compare_exchange_32(&schtStartFetch, 1, 0)) { @@ -431,13 +414,13 @@ void *schtFetchRspThread(void *aa) { } taosUsleep(1); - + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); - param->queryId = schtQueryId; + param->queryId = schtQueryId; param->taskId = schtFetchTaskId; - int32_t code = 0; + int32_t code = 0; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; @@ -446,7 +429,7 @@ void *schtFetchRspThread(void *aa) { dataBuf.len = sizeof(*rsp); code = schHandleCallback(param, &dataBuf, TDMT_SCH_FETCH_RSP, 0); - + assert(code == 0 || code); } return NULL; @@ -454,8 +437,8 @@ void *schtFetchRspThread(void *aa) { void schtFreeQueryJob(int32_t freeThread) { static uint32_t freeNum = 0; - int64_t job = queryJobRefId; - + int64_t job = queryJobRefId; + if (job && atomic_val_compare_exchange_64(&queryJobRefId, job, 0)) { schedulerFreeJob(&job, 0); if (freeThread) { @@ -466,31 +449,29 @@ void schtFreeQueryJob(int32_t freeThread) { } } -void* schtRunJobThread(void *aa) { - void *mockPointer = (void *)0x1; - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; +void *schtRunJobThread(void *aa) { + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - SQueryPlan dag; + SQueryPlan dag; schtInitLogFile(); - int32_t code = schedulerInit(); assert(code == 0); - schtSetPlanToString(); schtSetExecNode(); schtSetAsyncSendMsgToServer(); - SSchJob *pJob = NULL; + SSchJob *pJob = NULL; SSchTaskCallbackParam *param = NULL; - SHashObj *execTasks = NULL; - SDataBuf dataBuf = {0}; - uint32_t jobFinished = 0; - int32_t queryDone = 0; + SHashObj *execTasks = NULL; + SDataBuf dataBuf = {0}; + uint32_t jobFinished = 0; + int32_t queryDone = 0; while (!schtTestStop) { schtBuildQueryDag(&dag); @@ -503,10 +484,10 @@ void* schtRunJobThread(void *aa) { taosArrayPush(qnodeList, &qnodeAddr); queryDone = 0; - + SRequestConnInfo conn = {0}; conn.pTrans = mockPointer; - SSchedulerReq req = {0}; + SSchedulerReq req = {0}; req.syncReq = false; req.pConn = &conn; req.pNodeList = qnodeList; @@ -514,8 +495,8 @@ void* schtRunJobThread(void *aa) { req.sql = "select * from tb"; req.execFp = schtQueryCb; req.cbParam = &queryDone; - - code = schedulerExecJob(&req, &queryJobRefId); + + code = schedulerExecJob(&req, &queryJobRefId); assert(code == 0); pJob = schAcquireJob(queryJobRefId); @@ -524,20 +505,20 @@ void* schtRunJobThread(void *aa) { schtFreeQueryDag(&dag); continue; } - + execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; schtFetchTaskId = task->taskId - 1; - + taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task)); pIter = taosHashIterate(pJob->execTasks, pIter); - } + } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; - param->queryId = pJob->queryId; + param->queryId = pJob->queryId; pIter = taosHashIterate(execTasks, NULL); while (pIter) { @@ -547,16 +528,16 @@ void* schtRunJobThread(void *aa) { SQueryTableRsp rsp = {0}; dataBuf.pData = &rsp; dataBuf.len = sizeof(rsp); - + code = schHandleCallback(param, &dataBuf, TDMT_SCH_QUERY_RSP, 0); assert(code == 0 || code); pIter = taosHashIterate(execTasks, pIter); - } + } param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; - param->queryId = pJob->queryId; + param->queryId = pJob->queryId; pIter = taosHashIterate(execTasks, NULL); while (pIter) { @@ -566,28 +547,27 @@ void* schtRunJobThread(void *aa) { SQueryTableRsp rsp = {0}; dataBuf.pData = &rsp; dataBuf.len = sizeof(rsp); - + code = schHandleCallback(param, &dataBuf, TDMT_SCH_QUERY_RSP, 0); assert(code == 0 || code); - - pIter = taosHashIterate(execTasks, pIter); - } + pIter = taosHashIterate(execTasks, pIter); + } while (true) { if (queryDone) { break; } - + taosUsleep(10000); } atomic_store_32(&schtStartFetch, 1); - void *data = NULL; + void *data = NULL; req.syncReq = true; req.pFetchRes = &data; - + code = schedulerFetchRows(queryJobRefId, &req); assert(code == 0 || code); @@ -600,7 +580,7 @@ void* schtRunJobThread(void *aa) { data = NULL; code = schedulerFetchRows(queryJobRefId, &req); assert(code == 0 || code); - + schtFreeQueryJob(0); taosHashCleanup(execTasks); @@ -620,7 +600,7 @@ void* schtRunJobThread(void *aa) { return NULL; } -void* schtFreeJobThread(void *aa) { +void *schtFreeJobThread(void *aa) { while (!schtTestStop) { taosUsleep(taosRand() % 100); schtFreeQueryJob(1); @@ -628,17 +608,16 @@ void* schtFreeJobThread(void *aa) { return NULL; } - -} +} // namespace TEST(queryTest, normalCase) { - void *mockPointer = (void *)0x1; - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - int64_t job = 0; - SQueryPlan dag; + int64_t job = 0; + SQueryPlan dag; memset(&dag, 0, sizeof(dag)); @@ -648,7 +627,7 @@ TEST(queryTest, normalCase) { strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -662,30 +641,29 @@ TEST(queryTest, normalCase) { SRequestConnInfo conn = {0}; conn.pTrans = mockPointer; - SSchedulerReq req = {0}; + SSchedulerReq req = {0}; req.pConn = &conn; req.pNodeList = qnodeList; req.pDag = &dag; req.sql = "select * from tb"; req.execFp = schtQueryCb; req.cbParam = &queryDone; - - code = schedulerExecJob(&req, &job); + + code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); - + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); - + ASSERT_EQ(code, 0); pIter = taosHashIterate(pJob->execTasks, pIter); - } + } pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { @@ -693,10 +671,10 @@ TEST(queryTest, normalCase) { SQueryTableRsp rsp = {0}; code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); - + ASSERT_EQ(code, 0); pIter = taosHashIterate(pJob->execTasks, pIter); - } + } while (true) { if (queryDone) { @@ -705,14 +683,14 @@ TEST(queryTest, normalCase) { taosUsleep(10000); } - + TdThreadAttr thattr; taosThreadAttrInit(&thattr); TdThread thread1; taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); - void *data = NULL; + void *data = NULL; req.syncReq = true; req.pFetchRes = &data; @@ -739,13 +717,13 @@ TEST(queryTest, normalCase) { } TEST(queryTest, readyFirstCase) { - void *mockPointer = (void *)0x1; - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - int64_t job = 0; - SQueryPlan dag; + int64_t job = 0; + SQueryPlan dag; memset(&dag, 0, sizeof(dag)); @@ -755,7 +733,7 @@ TEST(queryTest, readyFirstCase) { strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -765,11 +743,11 @@ TEST(queryTest, readyFirstCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - int32_t queryDone = 0; + int32_t queryDone = 0; SRequestConnInfo conn = {0}; conn.pTrans = mockPointer; - SSchedulerReq req = {0}; + SSchedulerReq req = {0}; req.pConn = &conn; req.pNodeList = qnodeList; req.pDag = &dag; @@ -779,19 +757,18 @@ TEST(queryTest, readyFirstCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); - + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); - + ASSERT_EQ(code, 0); pIter = taosHashIterate(pJob->execTasks, pIter); - } + } pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { @@ -799,10 +776,10 @@ TEST(queryTest, readyFirstCase) { SQueryTableRsp rsp = {0}; code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); - + ASSERT_EQ(code, 0); pIter = taosHashIterate(pJob->execTasks, pIter); - } + } while (true) { if (queryDone) { @@ -812,14 +789,13 @@ TEST(queryTest, readyFirstCase) { taosUsleep(10000); } - TdThreadAttr thattr; taosThreadAttrInit(&thattr); TdThread thread1; taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); - void *data = NULL; + void *data = NULL; req.syncReq = true; req.pFetchRes = &data; code = schedulerFetchRows(job, &req); @@ -844,28 +820,26 @@ TEST(queryTest, readyFirstCase) { schedulerDestroy(); } - - TEST(queryTest, flowCtrlCase) { - void *mockPointer = (void *)0x1; - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - int64_t job = 0; - SQueryPlan dag; + int64_t job = 0; + SQueryPlan dag; schtInitLogFile(); taosSeedRand(taosGetTimestampSec()); - + SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); SEp qnodeAddr = {0}; strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -875,10 +849,10 @@ TEST(queryTest, flowCtrlCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - int32_t queryDone = 0; + int32_t queryDone = 0; SRequestConnInfo conn = {0}; conn.pTrans = mockPointer; - SSchedulerReq req = {0}; + SSchedulerReq req = {0}; req.pConn = &conn; req.pNodeList = qnodeList; req.pDag = &dag; @@ -889,17 +863,16 @@ TEST(queryTest, flowCtrlCase) { code = schedulerExecJob(&req, &job); ASSERT_EQ(code, 0); - SSchJob *pJob = schAcquireJob(job); bool qDone = false; - + while (!qDone) { void *pIter = taosHashIterate(pJob->execTasks, NULL); if (NULL == pIter) { break; } - + while (pIter) { SSchTask *task = *(SSchTask **)pIter; @@ -908,15 +881,15 @@ TEST(queryTest, flowCtrlCase) { if (task->lastMsgType == TDMT_SCH_QUERY) { SQueryTableRsp rsp = {0}; code = schHandleResponseMsg(pJob, task, TDMT_SCH_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); - + ASSERT_EQ(code, 0); } else { qDone = true; break; } - + pIter = NULL; - } + } } while (true) { @@ -933,7 +906,7 @@ TEST(queryTest, flowCtrlCase) { TdThread thread1; taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); - void *data = NULL; + void *data = NULL; req.syncReq = true; req.pFetchRes = &data; code = schedulerFetchRows(job, &req); @@ -958,15 +931,14 @@ TEST(queryTest, flowCtrlCase) { schedulerDestroy(); } - TEST(insertTest, normalCase) { - void *mockPointer = (void *)0x1; - char *clusterId = "cluster1"; - char *dbname = "1.db1"; - char *tablename = "table1"; + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - SQueryPlan dag; - uint64_t numOfRows = 0; + SQueryPlan dag; + uint64_t numOfRows = 0; SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); @@ -974,7 +946,7 @@ TEST(insertTest, normalCase) { strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -993,21 +965,21 @@ TEST(insertTest, normalCase) { SRequestConnInfo conn = {0}; conn.pTrans = mockPointer; - SSchedulerReq req = {0}; + SSchedulerReq req = {0}; req.pConn = &conn; req.pNodeList = qnodeList; req.pDag = &dag; req.sql = "insert into tb values(now,1)"; req.execFp = schtQueryCb; req.cbParam = NULL; - + code = schedulerExecJob(&req, &insertJobRefId); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); schedulerFreeJob(&insertJobRefId, 0); - schedulerDestroy(); + schedulerDestroy(); } TEST(multiThread, forceFree) { @@ -1027,12 +999,12 @@ TEST(multiThread, forceFree) { break; } } - + schtTestStop = true; taosSsleep(3); } -int main(int argc, char** argv) { +int main(int argc, char **argv) { taosSeedRand(taosGetTimestampSec()); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 4a63cd3bb28cdbb31ad4f2ca7531787fccb7e7d4..ee317d0751c90f4f63669a228ce35f640b5bde1b 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -67,7 +67,11 @@ void streamSchedByTimer(void* param, void* tmrId) { atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); - streamTaskInput(pTask, (SStreamQueueItem*)trigger); + if (streamTaskInput(pTask, (SStreamQueueItem*)trigger) < 0) { + taosFreeQitem(trigger); + taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); + return; + } streamSchedExec(pTask); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 63a28e17ed7461d8b0eb95c8eece7f31f0f43710..53e49a6ba5e0be669329c5ff6414c3648ee14fb3 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -28,6 +28,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF sprintf(streamPath, "%s/%s", path, "stream"); pMeta->path = strdup(streamPath); if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db) < 0) { + taosMemoryFree(streamPath); goto _err; } @@ -58,7 +59,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF return pMeta; _err: - if (pMeta->path) taosMemoryFree(pMeta->path); + taosMemoryFree(pMeta->path); if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks); if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb); if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb); @@ -250,6 +251,8 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { + tdbFree(pKey); + tdbFree(pVal); return -1; } tDecoderInit(&decoder, (uint8_t*)pVal, vLen); @@ -257,10 +260,14 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) { tDecoderClear(&decoder); if (pMeta->expandFunc(pMeta->ahandle, pTask) < 0) { + tdbFree(pKey); + tdbFree(pVal); return -1; } if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + tdbFree(pKey); + tdbFree(pVal); return -1; } } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index ea590c57e66f615872eecee0f66dce581dca52f5..c0ed135d74343395be1147fa72d012e032cd1f2b 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -63,7 +63,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int sprintf(statePath, "%s/%d", path, pTask->taskId); } else { memset(statePath, 0, 300); - strncpy(statePath, path, 300); + tstrncpy(statePath, path, 300); } if (tdbOpen(statePath, szPage, pages, &pState->db) < 0) { goto _err; diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 7c4aaba6ce117be00b00259254029bc9982db49e..5aadb599a30b0df5cc2b89608ea0afcfb79298c1 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -174,14 +174,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { if (res == TSDB_CODE_SUCCESS) { return false; } else { - qDebug("===stream===Update close window sbf. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 - ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } } - qDebug("===stream===Update close window. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } @@ -203,14 +198,10 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { } if (ts < pInfo->minTS) { - qDebug("===stream===Update min ts. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, - tableId, maxTs, *pMapMaxTs, ts); return true; } else if (res == TSDB_CODE_SUCCESS) { return false; } - qDebug("===stream===Update. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId, - maxTs, *pMapMaxTs, ts); // check from tsdb api return true; } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0afc373f2d0d90333002245e43817d12ef02df8c..101e99afea8db60011fa535a53320a32c2dd376c 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -169,7 +169,7 @@ typedef struct SSyncNode { } SSyncNode; // open/close -------------- -SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); +SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo); void syncNodeStart(SSyncNode* pSyncNode); void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index e000ba8bf869454a0276411d7312190be98348b9..4a35a15d3e35189daa291094c575e35d05ed8e02 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -579,6 +579,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -710,6 +711,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -859,6 +861,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -974,6 +977,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index b604d25816e300560571458fde7153196e77eee5..511113352ea07625b3cbf223add13d2c807ddbac 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -76,7 +76,10 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); } else { pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); - ASSERT(pEntry != NULL); + if (pEntry == NULL) { + sError("failed to get entry since %s. index:%lld", tstrerror(terrno), index); + return; + } } // cannot commit, even if quorum agree. need check term! if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) { @@ -127,7 +130,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { // execute fsm if (pSyncNode->pFsm != NULL) { int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state); - ASSERT(code == 0); + if (code != 0) { + wError("failed to commit sync node since %s", tstrerror(terrno)); + } } } } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 378ac743b33df0b78d042ac28deaecc0c573397f..907eb21f4c84a4b26169ef81beb9d272d1a14bce 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -86,10 +86,10 @@ void syncCleanUp() { } } -int64_t syncOpen(const SSyncInfo* pSyncInfo) { +int64_t syncOpen(SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); if (pSyncNode == NULL) { - sError("failed to open sync node. vgId:%d", pSyncInfo->vgId); + sError("vgId:%d, failed to open sync node since %s", pSyncInfo->vgId, terrstr()); return -1; } @@ -230,7 +230,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg return ret; } -int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { +int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; @@ -238,6 +238,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { } ASSERT(rid == pSyncNode->rid); +#if 0 if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; @@ -259,6 +260,12 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; +#else + syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg); + syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID); + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return 0; +#endif } int32_t syncLeaderTransfer(int64_t rid) { @@ -919,9 +926,7 @@ _END: } // open/close -------------- -SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { - SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo; - +SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = (SSyncNode*)taosMemoryCalloc(1, sizeof(SSyncNode)); if (pSyncNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -957,7 +962,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { sError("failed to open raft cfg file. path:%s", pSyncNode->configPath); goto _error; } - pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; + if (pSyncInfo->syncCfg.replicaNum == 0) { + pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; + } raftCfgClose(pSyncNode->pRaftCfg); pSyncNode->pRaftCfg = NULL; @@ -2650,7 +2657,10 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { if (ths->state == TAOS_SYNC_STATE_LEADER) { int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); - ASSERT(code == 0); + if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); + return -1; + } syncNodeReplicate(ths, false); } @@ -2726,7 +2736,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); if (code != 0) { // del resp mgr, call FpCommitCb - ASSERT(0); + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -2790,8 +2800,8 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); // del resp mgr, call FpCommitCb - ASSERT(0); return -1; } @@ -3043,7 +3053,10 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); } else { code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); - ASSERT(code == 0); + if (code != 0) { + sError("vgId:%d, failed to get log entry since %s. index:%lld", ths->vgId, tstrerror(terrno), i); + return -1; + } ASSERT(pEntry != NULL); } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 496c8419de7f56a96d544997f989ef6d16de4de3..c3dad104d1629cf127a1eae5034fdf0844930e74 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -234,8 +234,6 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pEntry->index, err, err, errStr, sysErr, sysErrStr); syncNodeErrorLog(pData->pSyncNode, logBuf); - - ASSERT(0); return -1; } pEntry->index = index; @@ -414,13 +412,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { return 0; } -SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { +SSyncRaftEntry* logStoreGetEntryWithoutLock(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { - taosThreadMutexLock(&(pData->mutex)); - // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); SWalReader* pWalHandle = pData->pWalHandle; ASSERT(pWalHandle != NULL); @@ -444,7 +440,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { } } while (0); - ASSERT(0); + sError("failed to read ver since %s. index:%lld", tstrerror(terrno), index); + return NULL; } SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); @@ -465,7 +462,6 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { terrno = saveErr; */ - taosThreadMutexUnlock(&(pData->mutex)); return pEntry; } else { @@ -473,6 +469,16 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { } } +SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { + SSyncLogStoreData* pData = pLogStore->data; + SSyncRaftEntry *pEntry = NULL; + + taosThreadMutexLock(&pData->mutex); + pEntry = logStoreGetEntryWithoutLock(pLogStore, index); + taosThreadMutexUnlock(&pData->mutex); + return pEntry; +} + int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a148c1e36bbdabb249f6dc64dc7c86c6934e641e..13b7e0ac0aef3640e0959d7b6cf1a7cd96040a72 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -121,6 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -483,9 +484,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN ret = tdbPagerWrite(pPager, pChild); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to write page since %s", terrstr()); + return -1; } // Copy the root page content to the child page @@ -556,8 +556,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pOlds[i]); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } } @@ -583,8 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pParent); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -719,8 +717,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pNews[iNew]); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } } @@ -937,7 +934,7 @@ static int tdbFetchOvflPage(SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt) // mark dirty ret = tdbPagerWrite(pBt->pPager, *ppOfp); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -987,7 +984,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const // fetch a new ofp and make it dirty SPgno pgno = 0; - SPage *ofp, *nextOfp; + SPage *ofp = NULL, *nextOfp = NULL; ret = tdbFetchOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { @@ -1942,7 +1939,7 @@ int tdbBtcDelete(SBTC *pBtc) { // drop the cell on the leaf ret = tdbPagerWrite(pPager, pBtc->pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -1964,7 +1961,7 @@ int tdbBtcDelete(SBTC *pBtc) { if (idx < nCells) { ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -2029,7 +2026,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // mark dirty ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index ea16e805628e69128eb548eb1a58084eb384282a..6c01348bc29261622469c5458963505fac388c5a 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -106,7 +106,7 @@ int32_t tdbBegin(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerBegin(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to begin pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerCommit(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to commit pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerAbort(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to abort pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 04711eb6a0af802f61cc39da3fba1282dcaf0c83..dd2416e5b80aa35271ce19560cd88b3bcb91b9fd 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -34,9 +34,9 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage); static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage); -static FORCE_INLINE int32_t pageCmpFn(const void *lhs, const void *rhs) { - SPage *pPageL = (SPage *)(((uint8_t *)lhs) - sizeof(SRBTreeNode)); - SPage *pPageR = (SPage *)(((uint8_t *)rhs) - sizeof(SRBTreeNode)); +static FORCE_INLINE int32_t pageCmpFn(const SRBTreeNode *lhs, const SRBTreeNode *rhs) { + SPage *pPageL = (SPage *)(((uint8_t *)lhs) - offsetof(SPage, node)); + SPage *pPageR = (SPage *)(((uint8_t *)rhs) - offsetof(SPage, node)); SPgno pgnoL = TDB_PAGE_PGNO(pPageL); SPgno pgnoR = TDB_PAGE_PGNO(pPageR); @@ -156,6 +156,7 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) { ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -210,7 +211,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page to journal since %s", tstrerror(terrno)); return -1; } } @@ -226,6 +227,8 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { // Open the journal pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (pPager->jfd < 0) { + tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -243,9 +246,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { // sync the journal file ret = tdbOsFSync(pPager->jfd); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } // loop to write the dirty pages to file @@ -255,7 +258,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage = (SPage *)pNode; ret = tdbPagerWritePageToDB(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; } } @@ -277,11 +280,25 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { tRBTreeCreate(&pPager->rbt, pageCmpFn); // sync the db file - tdbOsFSync(pPager->fd); + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } // remove the journal file - tdbOsClose(pPager->jfd); - tdbOsRemove(pPager->jFileName); + if (tdbOsClose(pPager->jfd) < 0) { + tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pPager->inTran = 0; return 0; @@ -297,14 +314,14 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 0, sync the journal file ret = tdbOsFSync(pPager->jfd); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755); if (jfd == NULL) { - return 0; + return -1; } ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); @@ -516,11 +533,15 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno)); if (ret < 0) { + tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -540,13 +561,15 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - ASSERT(0); + tdbError("failed to lseek due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName, pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -590,23 +613,38 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - ASSERT(0); + tdbError("failed to lseek fd due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); if (ret < 0) { - ASSERT(0); + tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - tdbOsFSync(pPager->fd); + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } tdbOsFree(pageBuf); - tdbOsClose(jfd); - tdbOsRemove(pPager->jFileName); + if (tdbOsClose(jfd) < 0) { + tdbError("failed to close jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } return 0; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index e5a783292dc56f88432093848d039df3c005a1e2..c82af0d0e9b68a3606718bdef47bb783c7289c54 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -100,8 +100,8 @@ void rpcCloseImpl(void* arg) { taosMemoryFree(pRpc); } -void* rpcMallocCont(int32_t contLen) { - int32_t size = contLen + TRANS_MSG_OVERHEAD; +void* rpcMallocCont(int64_t contLen) { + int64_t size = contLen + TRANS_MSG_OVERHEAD; char* start = taosMemoryCalloc(1, size); if (start == NULL) { tError("failed to malloc msg, size:%d", size); @@ -120,11 +120,11 @@ void rpcFreeCont(void* cont) { tTrace("rpc free cont:%p", (char*)cont - TRANS_MSG_OVERHEAD); } -void* rpcReallocCont(void* ptr, int32_t contLen) { +void* rpcReallocCont(void* ptr, int64_t contLen) { if (ptr == NULL) return rpcMallocCont(contLen); char* st = (char*)ptr - TRANS_MSG_OVERHEAD; - int32_t sz = contLen + TRANS_MSG_OVERHEAD; + int64_t sz = contLen + TRANS_MSG_OVERHEAD; st = taosMemoryRealloc(st, sz); if (st == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 83323725bed859b755490f56029bb1bab6f621d0..359b9824cf437159dafa8ed76f93e0a6231cba98 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -597,6 +597,7 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; + tDebug("%s conn %p alloc read buf", CONN_GET_INST_LABEL(conn), conn); transAllocBuffer(pBuf, buf); } static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { @@ -609,7 +610,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { if (nread > 0) { pBuf->len += nread; while (transReadComplete(pBuf)) { - tTrace("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn); + tDebug("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn); if (pBuf->invalid) { cliHandleExcept(conn); break; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 42079ff005bf7fac9802c2435fbb64ed8912c68d..f843c8559cae4e2b1595de93cb73c11ec5c77c14 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -203,7 +203,7 @@ bool transReadComplete(SConnBuffer* connBuf) { } int transSetConnOption(uv_tcp_t* stream) { - uv_tcp_nodelay(stream, 1); + uv_tcp_nodelay(stream, 0); int ret = uv_tcp_keepalive(stream, 5, 60); return ret; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 57bf8a34b3c9dc0d74516b456deab8eba27083f4..3616b4808f8869633bce34f1840644318489bdda 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -332,7 +332,10 @@ void uvOnSendCb(uv_write_t* req, int status) { if (status == 0) { tTrace("conn %p data already was written on stream", conn); if (!transQueueEmpty(&conn->srvMsgs)) { - SSvrMsg* msg = transQueuePop(&conn->srvMsgs); + SSvrMsg* msg = transQueuePop(&conn->srvMsgs); + STraceId* trace = &msg->msg.info.traceId; + tGDebug("conn %p write data out", conn); + destroySmsg(msg); // send cached data if (!transQueueEmpty(&conn->srvMsgs)) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e49a9631911e57d7f30cc66d5b1db4036d8e3dac..279f4dc6562d45b11240628821a7b2ba3c8e6eab 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -35,113 +35,251 @@ int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVe int64_t FORCE_INLINE walGetAppliedVer(SWal* pWal) { return pWal->vers.appliedVer; } -static FORCE_INLINE void walBuildMetaName(SWal* pWal, int metaVer, char* buf) { - sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); +static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { + return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); } -static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { +static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { + return sprintf(buf, "%s/meta-ver.tmp", pWal->path); +} + +static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - ASSERT(sz > 0); -#if 0 - for (int i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); - } -#endif + terrno = TSDB_CODE_SUCCESS; + ASSERT(fileIdx >= 0 && fileIdx < sz); - SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1); + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); int64_t fileSize = 0; taosStatFile(fnameStr, &fileSize, NULL); - int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize); - pLastFileInfo->fileSize = fileSize; - TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + // ensure size as non-negative + pFileInfo->fileSize = TMAX(0, pFileInfo->fileSize); + uint64_t magic = WAL_MAGIC; + int64_t walCkHeadSz = sizeof(SWalCkHead); + int64_t end = fileSize; + int64_t offset = 0; + int32_t capacity = 0; + int32_t readSize = 0; + char* buf = NULL; + char* found = NULL; + bool firstTrial = pFileInfo->fileSize < fileSize; + + // search for the valid last WAL entry, e.g. block by block + while (1) { + offset = (firstTrial) ? pFileInfo->fileSize : TMAX(0, end - WAL_SCAN_BUF_SIZE); + ASSERT(offset <= end); + readSize = end - offset; + capacity = readSize + sizeof(magic); + + int64_t limit = WAL_RECOV_SIZE_LIMIT; + if (limit < readSize) { + wError("vgId:%d, possibly corrupted WAL range exceeds size limit (i.e. %" PRId64 " bytes). offset:%" PRId64 + ", end:%" PRId64 ", file:%s", + pWal->cfg.vgId, limit, offset, end, fnameStr); + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + goto _err; + } - char* buf = taosMemoryMalloc(readSize + 5); - if (buf == NULL) { - taosCloseFile(&pFile); - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; - return -1; - } + void* ptr = taosMemoryRealloc(buf, capacity); + if (ptr == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + goto _err; + } + buf = ptr; - int64_t offset; - offset = taosLSeekFile(pFile, -readSize, SEEK_END); - if (readSize != taosReadFile(pFile, buf, readSize)) { - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } - char* found = NULL; - while (1) { + if (readSize != taosReadFile(pFile, buf, readSize)) { + wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + readSize, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + char* candidate = NULL; char* haystack = buf; - char* candidate; - while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { - // read and validate + + while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) { + // validate head + int64_t len = readSize - (candidate - buf); + if (len < walCkHeadSz) { + break; + } SWalCkHead* logContent = (SWalCkHead*)candidate; - if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { - found = candidate; + if (walValidHeadCksum(logContent) != 0) { + wError("vgId:%d, failed to validate checksum of wal entry header. offset:% %" PRId64 ", file:%s", + ((char*)(logContent)-buf), fnameStr); + haystack = candidate + 1; + if (firstTrial) { + break; + } else { + continue; + } } + + // validate body + int64_t size = walCkHeadSz + logContent->head.bodyLen; + if (len < size) { + int64_t extraSize = size - len; + if (capacity < readSize + extraSize + sizeof(magic)) { + capacity += extraSize; + void* ptr = taosMemoryRealloc(buf, capacity); + if (ptr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + buf = ptr; + } + int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), + offset); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s", + pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + } + if (walValidBodyCksum(logContent) != 0) { + terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; + wError("vgId:%d, failed to validate checksum of wal entry body. offset:% %" PRId64 ", file:%s", + ((char*)(logContent)-buf), fnameStr); + haystack = candidate + 1; + if (firstTrial) { + break; + } else { + continue; + } + } + + // found one + found = candidate; haystack = candidate + 1; } + if (found || offset == 0) break; - offset = TMIN(0, offset - readSize + sizeof(uint64_t)); - int64_t offset2 = taosLSeekFile(pFile, offset, SEEK_SET); - ASSERT(offset == offset2); - if (readSize != taosReadFile(pFile, buf, readSize)) { - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } -#if 0 - if (found == buf) { - SWalCkHead* logContent = (SWalCkHead*)found; - if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { - // file has to be deleted - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; - } - } -#endif + + // go backwards, e.g. by at most one WAL scan buf size + end = offset + walCkHeadSz - 1; + firstTrial = false; } - if (found == NULL) { - // file corrupted, no complete log - // TODO delete and search in previous files - /*ASSERT(0);*/ - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; + // determine end of last entry + SWalCkHead* lastEntry = (SWalCkHead*)found; + int64_t retVer = -1; + int64_t lastEntryBeginOffset = 0; + int64_t lastEntryEndOffset = 0; + + if (lastEntry == NULL) { + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + } else { + retVer = lastEntry->head.version; + lastEntryBeginOffset = offset + (int64_t)((char*)lastEntry - (char*)buf); + lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; } // truncate file - SWalCkHead* lastEntry = (SWalCkHead*)found; - int64_t retVer = lastEntry->head.version; - int64_t lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf); - int64_t lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; if (lastEntryEndOffset != fileSize) { - wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, + wWarn("vgId:%d, repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, fileSize); - taosFtruncateFile(pFile, lastEntryEndOffset); - ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset; - pWal->totSize -= (fileSize - lastEntryEndOffset); + if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) { + wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (taosFsyncFile(pFile) < 0) { + wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } } + pFileInfo->fileSize = lastEntryEndOffset; taosCloseFile(&pFile); taosMemoryFree(buf); - return retVer; + +_err: + taosCloseFile(&pFile); + taosMemoryFree(buf); + return -1; +} + +static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) { + int metaFileNum = taosArrayGetSize(metaLogList); + int actualFileNum = taosArrayGetSize(actualLogList); + int j = 0; + + // both of the lists in asc order + for (int i = 0; i < actualFileNum; i++) { + SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i); + while (j < metaFileNum) { + SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j); + ASSERT(pMetaInfo != NULL); + if (pMetaInfo->firstVer < pLogInfo->firstVer) { + j++; + } else if (pMetaInfo->firstVer == pLogInfo->firstVer) { + (*pLogInfo) = *pMetaInfo; + j++; + break; + } else { + break; + } + } + } + + taosArrayClear(metaLogList); + + for (int i = 0; i < actualFileNum; i++) { + SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i); + taosArrayPush(metaLogList, pFileInfo); + } +} + +void walAlignVersions(SWal* pWal) { + if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) { + wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId, + pWal->vers.firstVer, pWal->vers.snapshotVer); + pWal->vers.firstVer = pWal->vers.snapshotVer + 1; + } + if (pWal->vers.lastVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, + pWal->vers.lastVer, pWal->vers.snapshotVer); + pWal->vers.lastVer = pWal->vers.snapshotVer; + } + if (pWal->vers.commitVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, + pWal->vers.commitVer, pWal->vers.snapshotVer); + pWal->vers.commitVer = pWal->vers.snapshotVer; + } + if (pWal->vers.appliedVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, + pWal->vers.appliedVer, pWal->vers.snapshotVer); + pWal->vers.appliedVer = pWal->vers.snapshotVer; + } + + pWal->vers.commitVer = TMIN(pWal->vers.lastVer, pWal->vers.commitVer); + pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer); } int walCheckAndRepairMeta(SWal* pWal) { @@ -150,7 +288,6 @@ int walCheckAndRepairMeta(SWal* pWal) { const char* idxPattern = "^[0-9]+.idx$"; regex_t logRegPattern; regex_t idxRegPattern; - bool fixed = false; regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); @@ -184,222 +321,237 @@ int walCheckAndRepairMeta(SWal* pWal) { taosArraySort(actualLog, compareWalFileInfo); - int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); - int actualFileNum = taosArrayGetSize(actualLog); + int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); + int actualFileNum = taosArrayGetSize(actualLog); + int64_t firstVerPrev = pWal->vers.firstVer; + int64_t lastVerPrev = pWal->vers.lastVer; + int64_t totSize = 0; + bool updateMeta = (metaFileNum != actualFileNum); + + // rebuild meta of file info + walRebuildFileInfoSet(pWal->fileInfoSet, actualLog); + taosArrayDestroy(actualLog); -#if 0 - for (int32_t fileNo = actualFileNum - 1; fileNo >= 0; fileNo--) { - SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, fileNo); + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + ASSERT(sz == actualFileNum); + + // scan and determine the lastVer + int32_t fileIdx = sz; + + while (--fileIdx >= 0) { char fnameStr[WAL_FILE_LEN]; + int64_t fileSize = 0; + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - int64_t fileSize = 0; - taosStatFile(fnameStr, &fileSize, NULL); - if (fileSize == 0) { - taosRemoveFile(fnameStr); - walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); - taosRemoveFile(fnameStr); - taosArrayPop(pLogInfoArray); - } else { - break; + int32_t code = taosStatFile(fnameStr, &fileSize, NULL); + if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("failed to stat file since %s. file:%s", terrstr(), fnameStr); + return -1; } - } - actualFileNum = taosArrayGetSize(pLogInfoArray); -#endif - - { - int32_t i = 0, j = 0; - while (i < actualFileNum && j < metaFileNum) { - SWalFileInfo* pActualFile = taosArrayGet(actualLog, i); - SWalFileInfo* pMetaFile = taosArrayGet(pWal->fileInfoSet, j); - if (pActualFile->firstVer < pMetaFile->firstVer) { - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - i++; - } else if (pActualFile->firstVer > pMetaFile->firstVer) { - taosArrayRemove(pWal->fileInfoSet, j); - metaFileNum--; - } else { - i++; - j++; - } + ASSERT(pFileInfo->firstVer >= 0); + + if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { + totSize += pFileInfo->fileSize; + continue; } - if (i == actualFileNum && j == metaFileNum) { - if (j > 0) { - SWalFileInfo* pLastInfo = taosArrayGet(pWal->fileInfoSet, j - 1); - int64_t fsize = 0; - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pLastInfo->firstVer, fNameStr); - taosStatFile(fNameStr, &fsize, NULL); - if (pLastInfo->fileSize != fsize) { - fixed = true; - pLastInfo->fileSize = fsize; - pLastInfo->lastVer = walScanLogGetLastVer(pWal); - } - } - } else { - fixed = true; - while (i < actualFileNum) { - SWalFileInfo* pActualFile = taosArrayGet(actualLog, i); - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pActualFile->firstVer, fNameStr); - taosStatFile(fNameStr, &pActualFile->fileSize, NULL); - - if (pActualFile->fileSize == 0) { - ASSERT(i == actualFileNum - 1); - taosRemoveFile(fNameStr); - - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - break; - } + updateMeta = true; - if (i < actualFileNum - 1) { - pActualFile->lastVer = ((SWalFileInfo*)taosArrayGet(actualLog, i + 1))->firstVer - 1; - taosArrayPush(pWal->fileInfoSet, pActualFile); - i++; - } else { - pActualFile = taosArrayPush(pWal->fileInfoSet, pActualFile); - pActualFile->lastVer = walScanLogGetLastVer(pWal); - if (pActualFile->lastVer == -1) { - taosRemoveFile(fNameStr); - - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - taosArrayPop(pWal->fileInfoSet); - } - break; - } + int64_t lastVer = walScanLogGetLastVer(pWal, fileIdx); + if (lastVer < 0) { + if (terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { + wError("failed to scan wal last ver since %s", terrstr()); + return -1; } + ASSERT(pFileInfo->fileSize == 0); + // remove the empty wal log, and its idx + taosRemoveFile(fnameStr); + walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); + taosRemoveFile(fnameStr); + // remove its meta entry + taosArrayRemove(pWal->fileInfoSet, fileIdx); + continue; } - } -#if 0 - if (metaFileNum > actualFileNum) { - taosArrayPopFrontBatch(pWal->fileInfoSet, metaFileNum - actualFileNum); - } else if (metaFileNum < actualFileNum) { - for (int i = metaFileNum; i < actualFileNum; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(actualLog, i); - taosArrayPush(pWal->fileInfoSet, pFileInfo); - } + // update lastVer + pFileInfo->lastVer = lastVer; + totSize += pFileInfo->fileSize; } -#endif - - taosArrayDestroy(actualLog); + // reset vers info and so on actualFileNum = taosArrayGetSize(pWal->fileInfoSet); pWal->writeCur = actualFileNum - 1; - + pWal->totSize = totSize; + pWal->vers.lastVer = -1; if (actualFileNum > 0) { - int64_t fLastVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur))->lastVer; - if (fLastVer != -1 && pWal->vers.lastVer != fLastVer) { - fixed = true; - pWal->vers.lastVer = fLastVer; - } - int64_t fFirstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; - if (fFirstVer != pWal->vers.firstVer) { - fixed = true; - pWal->vers.firstVer = fFirstVer; - } + pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer; } + (void)walAlignVersions(pWal); - if (fixed) { - walSaveMeta(pWal); + // update meta file + if (updateMeta) { + (void)walSaveMeta(pWal); + } + return 0; +} + +int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { + if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (walValidHeadCksum(pCkHead) != 0) { + terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; + return -1; } return 0; } -int walCheckAndRepairIdx(SWal* pWal) { +int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - for (int32_t i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); - - char fnameStr[WAL_FILE_LEN]; - walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); - int64_t fsize; - TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); - if (pIdxFile == NULL) { - ASSERT(0); + ASSERT(fileIdx >= 0 && fileIdx < sz); + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); + char fLogNameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); + int64_t fileSize = 0; + + if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) { + wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer); + if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { + return 0; + } + + // start to repair + int64_t offset = fileSize - fileSize % sizeof(SWalIdxEntry); + TdFilePtr pLogFile = NULL; + TdFilePtr pIdxFile = NULL; + SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer - 1, .offset = -sizeof(SWalCkHead)}; + SWalCkHead ckHead; + memset(&ckHead, 0, sizeof(ckHead)); + ckHead.head.version = idxEntry.ver; + + pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); + if (pIdxFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); + if (pLogFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); + goto _err; + } + + // determine the last valid entry end, i.e. offset + while ((offset -= sizeof(SWalIdxEntry)) >= 0) { + if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) { + wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr()); - return -1; + goto _err; + } + + if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } - taosFStatFile(pIdxFile, &fsize, NULL); - if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { - taosCloseFile(&pIdxFile); + if (idxEntry.ver > pFileInfo->lastVer) { continue; } - int32_t left = fsize % sizeof(SWalIdxEntry); - int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END); - if (left != 0) { - taosFtruncateFile(pIdxFile, offset); - wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr, - offset, fsize); + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { + wWarn("vgId:%d, failed to read log file since %s. file:%s, offset:%" PRId64 ", idx entry ver:%" PRId64 "", + pWal->cfg.vgId, terrstr(), fLogNameStr, idxEntry.offset, idxEntry.ver); + continue; } - offset -= sizeof(SWalIdxEntry); - SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer}; - while (1) { - if (offset < 0) { - taosLSeekFile(pIdxFile, 0, SEEK_SET); - taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - break; - } - taosLSeekFile(pIdxFile, offset, SEEK_SET); - int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) { - taosFtruncateFile(pIdxFile, offset); - wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld", - pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset); - offset -= sizeof(SWalIdxEntry); - } else { - break; - } + if (idxEntry.ver == ckHead.head.version) { + break; } + } + offset += sizeof(SWalIdxEntry); - if (idxEntry.ver < pFileInfo->lastVer) { - char fLogNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); - TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); - if (pLogFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); - return -1; - } - while (idxEntry.ver < pFileInfo->lastVer) { - if (taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot seek file %s at %ld, since %s", pWal->cfg.vgId, fLogNameStr, idxEntry.offset, - terrstr()); - return -1; - } - SWalCkHead ckHead; - taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead)); - if (idxEntry.ver != ckHead.head.version) { - // todo truncate this idx also - taosCloseFile(&pLogFile); - wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId, - idxEntry.offset, idxEntry.ver, ckHead.head.version); - return -1; - } - idxEntry.ver = ckHead.head.version + 1; - idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen; - wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); - taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - } - taosCloseFile(&pLogFile); + // ftruncate idx file + if (offset < fileSize) { + if (taosFtruncateFile(pIdxFile, offset) < 0) { + wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + strerror(errno), offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } + + // rebuild idx file + if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) { + wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + while (idxEntry.ver < pFileInfo->lastVer) { + ASSERT(idxEntry.ver == ckHead.head.version); + + idxEntry.ver += 1; + idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; + + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { + wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), + idxEntry.offset, fLogNameStr); + goto _err; + } + wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); + if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { + wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); + goto _err; + } + } + + if (taosFsyncFile(pIdxFile) < 0) { + wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); + goto _err; + } + + (void)taosCloseFile(&pLogFile); + (void)taosCloseFile(&pIdxFile); + return 0; + +_err: + (void)taosCloseFile(&pLogFile); + (void)taosCloseFile(&pIdxFile); + return -1; +} + +int walCheckAndRepairIdx(SWal* pWal) { + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + int32_t fileIdx = sz; + while (--fileIdx >= 0) { + if (walCheckAndRepairIdxFile(pWal, fileIdx) < 0) { + wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx); + return -1; } - taosCloseFile(&pIdxFile); } return 0; } @@ -492,14 +644,20 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0); cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField; pRoot = cJSON_Parse(bytes); + if (!pRoot) goto _err; pMeta = cJSON_GetObjectItem(pRoot, "meta"); + if (!pMeta) goto _err; pField = cJSON_GetObjectItem(pMeta, "firstVer"); + if (!pField) goto _err; pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "snapshotVer"); + if (!pField) goto _err; pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "commitVer"); + if (!pField) goto _err; pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "lastVer"); + if (!pField) goto _err; pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField)); pFiles = cJSON_GetObjectItem(pRoot, "files"); @@ -509,17 +667,23 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { taosArrayEnsureCap(pArray, sz); SWalFileInfo* pData = pArray->pData; for (int i = 0; i < sz; i++) { - cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + if (!pInfoJson) goto _err; SWalFileInfo* pInfo = &pData[i]; pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); + if (!pField) goto _err; pInfo->firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "lastVer"); + if (!pField) goto _err; pInfo->lastVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "createTs"); + if (!pField) goto _err; pInfo->createTs = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "closeTs"); + if (!pField) goto _err; pInfo->closeTs = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "fileSize"); + if (!pField) goto _err; pInfo->fileSize = atoll(cJSON_GetStringValue(pField)); } taosArraySetSize(pArray, sz); @@ -527,6 +691,10 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pWal->writeCur = sz - 1; cJSON_Delete(pRoot); return 0; + +_err: + cJSON_Delete(pRoot); + return -1; } static int walFindCurMetaVer(SWal* pWal) { @@ -562,22 +730,63 @@ static int walFindCurMetaVer(SWal* pWal) { int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; - walBuildMetaName(pWal, metaVer + 1, fnameStr); - TdFilePtr pMetaFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE); + char tmpFnameStr[WAL_FILE_LEN]; + int n; + + // fsync the idx and log file at first to ensure validity of meta + if (taosFsyncFile(pWal->pIdxFile) < 0) { + wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (taosFsyncFile(pWal->pLogFile) < 0) { + wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + // flush to a tmpfile + n = walBuildTmpMetaName(pWal, tmpFnameStr); + ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name"); + + TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pMetaFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); if (len != taosWriteFile(pMetaFile, serialized, len)) { - // TODO:clean file + wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } - taosCloseFile(&pMetaFile); - taosRemoveFile(fnameStr); - return -1; + if (taosFsyncFile(pMetaFile) < 0) { + wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosCloseFile(&pMetaFile) < 0) { + wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // rename it + n = walBuildMetaName(pWal, metaVer + 1, fnameStr); + ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name"); + + if (taosRenameFile(tmpFnameStr, fnameStr) < 0) { + wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } - taosCloseFile(&pMetaFile); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -585,6 +794,11 @@ int walSaveMeta(SWal* pWal) { } taosMemoryFree(serialized); return 0; + +_err: + taosCloseFile(&pMetaFile); + taosMemoryFree(serialized); + return -1; } int walLoadMeta(SWal* pWal) { @@ -626,6 +840,10 @@ int walLoadMeta(SWal* pWal) { } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); + if (code < 0) { + wError("failed to deserialize wal meta. file:%s", fnameStr); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + } taosCloseFile(&pFile); taosMemoryFree(buf); return code; @@ -636,6 +854,5 @@ int walRemoveMeta(SWal* pWal) { if (metaVer == -1) return 0; char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); - taosRemoveFile(fnameStr); - return 0; + return taosRemoveFile(fnameStr); } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 7974f3e32e52b7434086bf438ba84c6e9772407b..29058cada1ce665882606ee186074a44452d6533 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -81,6 +81,12 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } + if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pWal); + return NULL; + } + // set config memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); @@ -98,15 +104,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { tstrncpy(pWal->path, path, sizeof(pWal->path)); if (taosMkDir(pWal->path) != 0) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - taosMemoryFree(pWal); - return NULL; + goto _err; } // init ref pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pWal->pRefHash == NULL) { - taosMemoryFree(pWal); - return NULL; + wError("failed to init hash since %s", tstrerror(terrno)); + goto _err; } // open meta @@ -117,9 +122,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - taosHashCleanup(pWal->pRefHash); - taosMemoryFree(pWal); - return NULL; + goto _err; } // init status @@ -131,46 +134,37 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.head.protoVer = WAL_PROTO_VER; pWal->writeHead.magic = WAL_MAGIC; - if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { - taosArrayDestroy(pWal->fileInfoSet); - taosHashCleanup(pWal->pRefHash); - taosMemoryFree(pWal); - return NULL; - } - - pWal->refId = taosAddRef(tsWal.refSetId, pWal); - if (pWal->refId < 0) { - taosHashCleanup(pWal->pRefHash); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - taosMemoryFree(pWal); - return NULL; - } - - walLoadMeta(pWal); + // load meta + (void)walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId); - taosHashCleanup(pWal->pRefHash); - taosRemoveRef(tsWal.refSetId, pWal->refId); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - return NULL; + goto _err; } if (walCheckAndRepairIdx(pWal) < 0) { wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId); - taosHashCleanup(pWal->pRefHash); - taosRemoveRef(tsWal.refSetId, pWal->refId); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - return NULL; + goto _err; + } + + // add ref + pWal->refId = taosAddRef(tsWal.refSetId, pWal); + if (pWal->refId < 0) { + wError("failed to add ref for Wal since %s", tstrerror(terrno)); + goto _err; } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); - return pWal; + +_err: + taosArrayDestroy(pWal->fileInfoSet); + taosHashCleanup(pWal->pRefHash); + taosThreadMutexDestroy(&pWal->mutex); + taosMemoryFree(pWal); + pWal = NULL; + return NULL; } int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { @@ -195,11 +189,11 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { void walClose(SWal *pWal) { taosThreadMutexLock(&pWal->mutex); + (void)walSaveMeta(pWal); taosCloseFile(&pWal->pLogFile); pWal->pLogFile = NULL; taosCloseFile(&pWal->pIdxFile); pWal->pIdxFile = NULL; - walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; taosHashCleanup(pWal->pRefHash); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index c4f1a81eeb10a46594efc9e2c4bfa89228b16ed6..179d809c84c5773c138ae7dc35284d5b3fb461eb 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -181,7 +181,11 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(pRet != NULL); + if (pRet == NULL) { + wError("failed to find WAL log file with ver:%lld", ver); + terrno = TSDB_CODE_WAL_INVALID_VER; + return -1; + } if (pReader->curFileFirstVer != pRet->firstVer) { // error code was set inner if (walReadChangeFile(pReader, pRet->firstVer) < 0) { @@ -472,7 +476,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - ASSERT(0); + wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -505,6 +510,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } + wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -514,7 +521,6 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { pReader->pHead->head.version, ver); pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -528,7 +534,6 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum); pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(0); taosThreadMutexUnlock(&pReader->mutex); return -1; } diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 1196914daea766159f20b507d2bfb908d6c309e0..4b75db52b710fd5a084ba1c13f5270006d385967 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -79,6 +79,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; if (pWal->pLogFile != NULL) { + code = taosFsyncFile(pWal->pLogFile); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } code = taosCloseFile(&pWal->pLogFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -86,6 +91,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { } } if (pWal->pIdxFile != NULL) { + code = taosFsyncFile(pWal->pIdxFile); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } code = taosCloseFile(&pWal->pIdxFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 628b432446dd8e1d650b07d0a00d16ed108ec5fb..91fa49fce0777b4f382e58878bc6c20f8f233beb 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -42,10 +42,22 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { SWalFileInfo *pFileInfo = taosArrayGet(pWal->fileInfoSet, i); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - taosRemoveFile(fnameStr); + if (taosRemoveFile(fnameStr) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); - taosRemoveFile(fnameStr); + if (taosRemoveFile(fnameStr) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } + wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); } } walRemoveMeta(pWal); @@ -209,10 +221,12 @@ int32_t walRollback(SWal *pWal, int64_t ver) { taosCloseFile(&pIdxFile); taosCloseFile(&pLogFile); - taosFsyncFile(pWal->pLogFile); - taosFsyncFile(pWal->pIdxFile); - - walSaveMeta(pWal); + code = walSaveMeta(pWal); + if (code < 0) { + wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } // unlock taosThreadMutexUnlock(&pWal->mutex); @@ -384,7 +398,11 @@ int32_t walRollImpl(SWal *pWal) { pWal->lastRollSeq = walGetSeq(); - walSaveMeta(pWal); + code = walSaveMeta(pWal); + if (code < 0) { + wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); + goto END; + } END: return code; @@ -540,6 +558,11 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { + wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); + if (taosFsyncFile(pWal->pIdxFile) < 0) { + wError("vgId:%d, file:%" PRId64 ".idx, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), + strerror(errno)); + } wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (taosFsyncFile(pWal->pLogFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index f0442c6fd12a352bc41c36104ed87551de8236e2..616ab7875dd9634b4ad4258c51320b589dd744ae 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -87,11 +87,17 @@ void osUpdate() { void osCleanup() {} -bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } +bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; } -bool osDataSpaceAvailable() { return tsDataSpace.reserved <= tsDataSpace.size.avail; } +bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; } -bool osTempSpaceAvailable() { return tsTempSpace.reserved <= tsTempSpace.size.avail; } +bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; } + +bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; } + +bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; } + +bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; } void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); } diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 8f026f68907f16953ef8e1149ccaaf908dab2f3a..75a963d5c7551ad290d1d2190c24ab07e1ed0953 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -228,7 +228,7 @@ void taosPrintBackTrace() { void taosPrintBackTrace() { return; } #endif -void *taosMemoryMalloc(int32_t size) { +void *taosMemoryMalloc(int64_t size) { #ifdef USE_TD_MEMORY void *tmp = malloc(size + sizeof(TdMemoryInfo)); if (tmp == NULL) return NULL; @@ -244,7 +244,7 @@ void *taosMemoryMalloc(int32_t size) { #endif } -void *taosMemoryCalloc(int32_t num, int32_t size) { +void *taosMemoryCalloc(int64_t num, int64_t size) { #ifdef USE_TD_MEMORY int32_t memorySize = num * size; char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1); @@ -261,7 +261,7 @@ void *taosMemoryCalloc(int32_t num, int32_t size) { #endif } -void *taosMemoryRealloc(void *ptr, int32_t size) { +void *taosMemoryRealloc(void *ptr, int64_t size) { #ifdef USE_TD_MEMORY if (ptr == NULL) return taosMemoryMalloc(size); @@ -318,7 +318,7 @@ void taosMemoryFree(void *ptr) { #endif } -int32_t taosMemorySize(void *ptr) { +int64_t taosMemorySize(void *ptr) { if (ptr == NULL) return 0; #ifdef USE_TD_MEMORY diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 7e1506c14071e9522eb45e10d9d6309eff217fa0..f3ccbb0aac5e06673681181795cf5ae1e30a43a8 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -19,12 +19,12 @@ #include "taos.h" #include "taoserror.h" -#define UNIT_NUM_BITS 64 -#define UNIT_ADDR_NUM_BITS 6 +#define UNIT_NUM_BITS 64ULL +#define UNIT_ADDR_NUM_BITS 6ULL static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1 << (index % UNIT_NUM_BITS); + uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); uint64_t old = buf[unitIndex]; buf[unitIndex] |= mask; return buf[unitIndex] != old; @@ -32,7 +32,7 @@ static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1 << (index % UNIT_NUM_BITS); + uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); return buf[unitIndex] & mask; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index eb13a08be442ea0365a5e71d01cf1e5f1d999d04..9a117c6eb48c1bc06d4063101531579ba6369fb4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -447,12 +447,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND, "TQ table schema not f TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no commited offset") // wal -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Wal unexpected generic error") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "WAL unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") // tfs TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 65f1bac60aa51c48fe65895dbe0ded241d22590d..cbbf53d8f1c18e4770c513bc678fc68646268ff7 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -219,7 +219,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { while (temp != pTree->NIL) { y = temp; - int32_t c = pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(temp)); + int32_t c = pTree->cmprFn(z, temp); if (c < 0) { temp = temp->left; } else if (c > 0) { @@ -232,7 +232,7 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { if (y == pTree->NIL) { pTree->root = z; - } else if (pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(y)) < 0) { + } else if (pTree->cmprFn(z, y) < 0) { y->left = z; } else { y->right = z; @@ -245,10 +245,10 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { tRBTreePutFix(pTree, z); // update min/max node - if (pTree->min == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->min), RBTREE_NODE_PAYLOAD(z)) > 0) { + if (pTree->min == pTree->NIL || pTree->cmprFn(pTree->min, z) > 0) { pTree->min = z; } - if (pTree->max == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->max), RBTREE_NODE_PAYLOAD(z)) < 0) { + if (pTree->max == pTree->NIL || pTree->cmprFn(pTree->max, z) < 0) { pTree->max = z; } pTree->n++; @@ -309,11 +309,11 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { return pNode; } -SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { +SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode) { SRBTreeNode *pNode = pTree->root; while (pNode != pTree->NIL) { - int32_t c = pTree->cmprFn(pKey, RBTREE_NODE_PAYLOAD(pNode)); + int32_t c = pTree->cmprFn(pKeyNode, pNode); if (c < 0) { pNode = pNode->left; diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 4ce668e6bab22abc74f509114ab84b0693469634..d93e9fc5695904995041307b1ec333f780ff2b6f 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -145,7 +145,10 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it tSkipListWLock(pSkipList); void *pData = iterate(iter); - if (pData == NULL) return; + if (pData == NULL) { + tSkipListUnlock(pSkipList); + return; + } // backward to put the first data hasDup = tSkipListGetPosToPut(pSkipList, backward, pData); diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py index 0a835ec5643d8901db475dddcdaf60d0baf317c9..c68375894c6f34bdccc33789384f6113aeb6e105 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py @@ -32,6 +32,10 @@ class TDTestCase: if "community" in selfPath: projPath = selfPath[: selfPath.find("community")] + elif "src" in selfPath: + projPath = selfPath[: selfPath.find("src")] + elif "/tools/" in selfPath: + projPath = selfPath[: selfPath.find("/tools/")] else: projPath = selfPath[: selfPath.find("tests")] @@ -55,11 +59,10 @@ class TDTestCase: tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.execute("reset query cache") - tdSql.query("select count(*) from (select distinct(tbname) from db.stb)") - tdSql.checkData(0, 0, 10) + tdSql.query("show db.tables") + tdSql.checkRows(10) tdSql.query("select count(*) from db.stb") - if len(tdSql.queryResult): - tdLog.exit("query result is %d" % len(tdSql.queryResult)) + tdSql.checkData(0, 0, 100) def stop(self): tdSql.close() diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json index d4b2aae2fb7bf1a3612e60f395a1c5ed26ceca8f..da22ef75e2a648e897341338bb762e4fedc6662b 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json @@ -20,8 +20,10 @@ "super_tables": [{ "name": "stb", "childtable_prefix": "stb_", + "childtable_count": 10, + "insert_rows": 10, "columns": [{"type": "INT"}], "tags": [{"type": "INT"}] }] }] -} \ No newline at end of file +} diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..457c9eae20795d03cc0b63de7f81df68cb91d735 --- /dev/null +++ b/tests/script/coverage_test.sh @@ -0,0 +1,291 @@ +#!/bin/bash + +branch= +if [ x$1 != x ];then + branch=$1 + echo "Testing branch: $branch" +else + echo "Please enter branch name as a parameter" + exit 1 +fi + +today=`date +"%Y%m%d"` +TDENGINE_DIR=/root/pxiao/TDengine +TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function buildTDengine { + echo "check if TDengine need build" + cd $TDENGINE_DIR + git remote prune origin > /dev/null + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + + # reset counter + lcov -d . --zerocounters + + + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + fi + + git reset --hard + git checkout -- . + git checkout $branch + git clean -dfx + git pull + git submodule update --init --recursive -f + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + + echo "rebuild.." + LOCAL_COMMIT=`git rev-parse --short @` + + rm -rf * + if [ "$branch" == "3.0" ]; then + echo "3.0 =============" + cmake -DCOVER=true -DBUILD_TEST=true .. + else + cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null + fi + make -j4 + make install +} + +function runGeneralCaseOneByOne { + while read -r line; do + if [[ $line =~ ^./test.sh* ]]; then + case=`echo $line | grep sim$ | awk '{print $NF}'` + + if [ -n "$case" ]; then + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTestNGCaseOneByOne { + while read -r line; do + if [[ $line =~ ^taostest* ]]; then + case=`echo $line | cut -d' ' -f 3 | cut -d'=' -f 2` + yaml=`echo $line | cut -d' ' -f 2` + + if [ -n "$case" ]; then + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && taostest $yaml --case=$case --keep --disable_collection > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTest { + echo "run Test" + + if [ "$branch" == "3.0" ]; then + echo "start run unit test case ................" + echo " $TDENGINE_DIR/debug " + cd $TDENGINE_DIR/debug + ctest -j12 + echo "3.0 unit test done" + fi + + cd $TDENGINE_DIR/tests/script + + [ -d ../../sim ] && rm -rf ../../sim + [ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT + + runGeneralCaseOneByOne jenkins/basic.txt + + sed -i "1i\SIM cases test result" $TDENGINE_COVERAGE_REPORT + + totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + sed -i -e "2i\ ### Total $totalSuccess SIM test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT + fi + + totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + sed -i "3i\### Total $totalFailed SIM test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT + fi + sed "3G" $TDENGINE_COVERAGE_REPORT + + stopTaosd + echo "run TestNG cases" + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 & + sleep 10 + cd $TDENGINE_DIR/../TestNG/cases + runTestNGCaseOneByOne ../scripts/cases.txt + echo "TestNG cases done" + + cd $TDENGINE_DIR/tests + rm -rf ../sim + /root/pxiao/test-all-coverage.sh full python $branch | tee -a $TDENGINE_COVERAGE_REPORT + + + sed -i "4i\Python cases test result" $TDENGINE_COVERAGE_REPORT + totalPySuccess=`grep 'python case(s) succeed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'` + if [ "$totalPySuccess" -gt "0" ]; then + sed -i -e "5i\ ### Total $totalPySuccess Python test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT + fi + + totalPyFailed=`grep 'python case(s) failed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'` + if [ -z $totalPyFailed ]; then + sed -i "6i\\n" $TDENGINE_COVERAGE_REPORT + else + sed -i "6i\### Total $totalPyFailed Python test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT + fi + + echo "### run JDBC test cases ###" | tee -a $TDENGINE_COVERAGE_REPORT + # Test Connector + stopTaosd + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 & + sleep 10 + + cd $TDENGINE_DIR/src/connector/jdbc + mvn clean package > /dev/null 2>&1 + mvn test > jdbc-out.log 2>&1 + tail -n 20 jdbc-out.log 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test C Demo + stopTaosd + $TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null & + sleep 10 + yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test waltest + dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'` + walDir=`find $dataDir -name "wal"|head -n1` + echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT + echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT + if [ -n "$walDir" ]; then + yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + fi + + # run Unit Test + echo "Run Unit Test: utilTest, queryTest and cliTest" + #$TDENGINE_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!" + #$TDENGINE_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!" + #$TDENGINE_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!" + + stopTaosd + + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDENGINE_DIR + + # collect data + lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + + # remove exclude paths + if [ "$branch" == "3.0" ]; then + lcov --remove coverage.info \ + '*/contrib/*' '*/tests/*' '*/test/*'\ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\ + '*/tthread.c' '*/tversion.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + else + lcov --remove coverage.info \ + '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + fi + + + # generate result + echo "generate result" + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + + # push result to coveralls.io + echo "push result to coveralls.io" + /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + + #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT + #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT + cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT + +} + +function sendReport { + echo "send report" + receiver="develop@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDENGINE_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT + BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT` + echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \ + /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" +} + +function stopTaosd { + echo "Stop taosd start" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + echo "Stop tasod end" +} + +function runTestRandomFail { + exec_random_fail_sh=$1 + default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh + [ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1 + + dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py + default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py + [ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1 + + runTest NoRandomFail +} + +WORK_DIR=/root/pxiao + +date >> $WORK_DIR/cron.log +echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log + +stopTaosd +buildTDengine + +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh + +runTest + +lcovFunc +#sendReport +stopTaosd + +date >> $WORK_DIR/cron.log +echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 82f73a4fdd209e84d85d8616ece777cf37e02dfb..d8f4a36261d3607d91b89d42aa06fbbd1b01ab57 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -155,7 +155,7 @@ ./test.sh -f tsim/parser/select_with_tags.sim ./test.sh -f tsim/parser/selectResNum.sim ./test.sh -f tsim/parser/set_tag_vals.sim -./test.sh -f tsim/parser/single_row_in_tb.sim +# TD-19572 ./test.sh -f tsim/parser/single_row_in_tb.sim ./test.sh -f tsim/parser/sliding.sim ./test.sh -f tsim/parser/slimit_alter_tags.sim ./test.sh -f tsim/parser/slimit.sim @@ -194,7 +194,7 @@ ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim -# TD-17919 ./test.sh -f tsim/mnode/basic4.sim +./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim # ---- show ---- @@ -352,6 +352,7 @@ ./test.sh -f tsim/scalar/in.sim ./test.sh -f tsim/scalar/scalar.sim ./test.sh -f tsim/scalar/filter.sim +./test.sh -f tsim/scalar/caseWhen.sim # ---- alter ---- ./test.sh -f tsim/alter/cached_schema_after_alter.sim diff --git a/tests/script/tsim/mnode/basic1.sim b/tests/script/tsim/mnode/basic1.sim index 59156080c823c9b54b29ce7da864e04198ca5aac..31fc276298e85bdc852ced81f5e1c765058378f8 100644 --- a/tests/script/tsim/mnode/basic1.sim +++ b/tests/script/tsim/mnode/basic1.sim @@ -42,6 +42,7 @@ sql_error drop mnode on dnode 1 print =============== create mnode 2 sql create mnode on dnode 2 +print =============== create mnode 2 finished $x = 0 step2: $x = $x + 1 @@ -69,9 +70,10 @@ if $data(2)[2] != follower then goto step2 endi -sleep 2000 print ============ drop mnode 2 sql drop mnode on dnode 2 + +print ============ drop mnode 2 finished sql select * from information_schema.ins_mnodes if $rows != 1 then return -1 @@ -109,6 +111,8 @@ sleep 2000 print =============== create mnodes sql create mnode on dnode 2 + +print =============== create mnode 2 finished sql select * from information_schema.ins_mnodes if $rows != 2 then return -1 diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim index a2b8aa2f967a6c8e56ed4281b25cbef3400615d2..6739a87b4233f4b57f70d886494ff5a8c55f7058 100644 --- a/tests/script/tsim/mnode/basic4.sim +++ b/tests/script/tsim/mnode/basic4.sim @@ -45,49 +45,12 @@ endi if $data(2)[4] != ready then goto step2 endi - -system sh/exec.sh -n dnode3 -s stop -sql_error create mnode on dnode 3 - -print =============== step3: select * from information_schema.ins_mnodes - -$x = 0 -step3: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi -sql select * from information_schema.ins_mnodes -x step3 -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] -print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] - -$leaderNum = 0 -if $data(1)[2] == leader then - $leaderNum = 1 -endi -if $data(2)[2] == leader then - $leaderNum = 1 -endi -if $leaderNum == 0 then - goto step3 -endi -if $data(3)[2] != offline then - goto step3 -endi -if $data(1)[3] != ready then - goto step3 -endi -if $data(2)[3] != ready then - goto step3 -endi -if $data(3)[3] != creating then - goto step3 +if $data(3)[4] != ready then + goto step2 endi -print =============== step4: start dnode3 -system sh/exec.sh -n dnode3 -s start +print =============== step4: create mnode 3 +sql create mnode on dnode 3 $x = 0 step4: @@ -159,12 +122,11 @@ endi if $data(2)[3] != ready then goto step5 endi -if $data(3)[3] != dropping then - goto step5 -endi print =============== step6: start dnode3 system sh/exec.sh -n dnode3 -s start +sql drop mnode on dnode 1 -x step60 +step60: $x = 0 step6: diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index 387f38a71793a0d6dad928d398db8ebdfab97692..16e8fa3dfaf9d3b8254de7e013e2383ee17a9e01 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -232,12 +232,11 @@ endi if $leaderNum != 1 then goto step81 endi -if $data(1)[3] != dropping then - goto step81 -endi print =============== step9: start mnode1 and wait it dropped system sh/exec.sh -n dnode1 -s start +sql drop mnode on dnode 1 -x step90 +step90: $x = 0 step91: diff --git a/tests/script/tsim/scalar/caseWhen.sim b/tests/script/tsim/scalar/caseWhen.sim new file mode 100644 index 0000000000000000000000000000000000000000..f6b9c3ff081a69a16851b2f9fee6cd0d5c3935c3 --- /dev/null +++ b/tests/script/tsim/scalar/caseWhen.sim @@ -0,0 +1,1064 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== prepare data + +sql drop database if exists db1; +sql create database db1 vgroups 5; +sql use db1; +sql create stable sta (ts timestamp, f1 int, f2 binary(10), f3 bool) tags(t1 int, t2 bool, t3 binary(10)); +sql create table tba1 using sta tags(0, false, '0'); +sql create table tba2 using sta tags(1, true, '1'); +sql create table tba3 using sta tags(null, null, ''); +sql create table tba4 using sta tags(1, false, null); +sql create table tba5 using sta tags(3, true, 'aa'); +sql insert into tba1 values ('2022-09-26 15:15:01', 0, "a", false); +sql insert into tba1 values ('2022-09-26 15:15:02', 1, "0", true); +sql insert into tba1 values ('2022-09-26 15:15:03', 5, "5", false); +sql insert into tba1 values ('2022-09-26 15:15:04', null, null, null); +sql insert into tba2 values ('2022-09-27 15:15:01', 0, "a", false); +sql insert into tba2 values ('2022-09-27 15:15:02', 1, "0", true); +sql insert into tba2 values ('2022-09-27 15:15:03', 5, "5", false); +sql insert into tba2 values ('2022-09-27 15:15:04', null, null, null); +sql insert into tba3 values ('2022-09-28 15:15:01', 0, "a", false); +sql insert into tba3 values ('2022-09-28 15:15:02', 1, "0", true); +sql insert into tba3 values ('2022-09-28 15:15:03', 5, "5", false); +sql insert into tba3 values ('2022-09-28 15:15:04', null, null, null); +sql insert into tba4 values ('2022-09-29 15:15:01', 0, "a", false); +sql insert into tba4 values ('2022-09-29 15:15:02', 1, "0", true); +sql insert into tba4 values ('2022-09-29 15:15:03', 5, "5", false); +sql insert into tba4 values ('2022-09-29 15:15:04', null, null, null); +sql insert into tba5 values ('2022-09-30 15:15:01', 0, "a", false); +sql insert into tba5 values ('2022-09-30 15:15:02', 1, "0", true); +sql insert into tba5 values ('2022-09-30 15:15:03', 5, "5", false); +sql insert into tba5 values ('2022-09-30 15:15:04', null, null, null); + +print ======== case when xx + +sql select case when 3 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 4 then + return -1 +endi +if $data20 != 4 then + return -1 +endi +if $data30 != 4 then + return -1 +endi + +sql select case when 0 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when null then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when 1 then 4+1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 5.000000000 then + return -1 +endi +if $data10 != 5.000000000 then + return -1 +endi +if $data20 != 5.000000000 then + return -1 +endi +if $data30 != 5.000000000 then + return -1 +endi + +sql select case when 1-1 then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when 1+1 then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 0 then + return -1 +endi +if $data30 != 0 then + return -1 +endi + +sql select case when 1 then 1-1+2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi +if $data10 != 2.000000000 then + return -1 +endi +if $data20 != 2.000000000 then + return -1 +endi +if $data30 != 2.000000000 then + return -1 +endi + +sql select case when 1 > 0 then 1 < 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case when 1 > 2 then 1 < 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when abs(3) then abs(-1) end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case when abs(1+1) then abs(-1)+abs(3) end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4.000000000 then + return -1 +endi +if $data10 != 4.000000000 then + return -1 +endi +if $data20 != 4.000000000 then + return -1 +endi +if $data30 != 4.000000000 then + return -1 +endi + +sql select case when 0 then 1 else 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 3 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 3 then + return -1 +endi + +sql select case when 0 then 1 when 1 then 0 else 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 0 then + return -1 +endi +if $data30 != 0 then + return -1 +endi + +sql select case when 0 then 1 when 1 then 0 when 2 then 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 0 then + return -1 +endi +if $data30 != 0 then + return -1 +endi + +sql select case when 'a' then 'b' when null then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when '2' then 'b' when null then 0 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != b then + return -1 +endi +if $data10 != b then + return -1 +endi +if $data20 != b then + return -1 +endi +if $data30 != b then + return -1 +endi + +sql select case when 0 then 'b' else null end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when 0 then 'b' else 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data20 != 2 then + return -1 +endi +if $data30 != 2 then + return -1 +endi + +sql select case when sum(2) then sum(2)-sum(1) end from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 4.000000000 then + return -1 +endi + +sql select case when sum(2) then abs(-2) end from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 2 then + return -1 +endi + +sql select case when ts then ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != @22-09-26 15:15:01.000@ then + return -1 +endi +if $data10 != @22-09-26 15:15:02.000@ then + return -1 +endi +if $data20 != @22-09-26 15:15:03.000@ then + return -1 +endi +if $data30 != @22-09-26 15:15:04.000@ then + return -1 +endi + +sql select case when f1 then ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != @22-09-26 15:15:02.000@ then + return -1 +endi +if $data20 != @22-09-26 15:15:03.000@ then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when f1 then f1 when f1 + 1 then f1 + 1 else f1 is null end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case when f1 then 3 when ts then ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1664176501000 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 1664176504000 then + return -1 +endi + +sql select case when 3 then f1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case when f1 then 3 when 1 then 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 2 then + return -1 +endi + +sql select case when sum(f1) then sum(f1)-abs(-1) end from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 5.000000000 then + return -1 +endi + +sql select case when sum(f1) then sum(f1)-abs(f1) end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != 0.000000000 then + return -1 +endi +if $data30 != 0.000000000 then + return -1 +endi + +sql select case when f1 then sum(f1) when f1 is not null then 9 else 8 end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != 8 then + return -1 +endi +if $data10 != 9 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 5 then + return -1 +endi + +sql select f1 from tba1 where f1 > case when f1 then 0 else 3 end; +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 5 then + return -1 +endi + +sql select f1 from tba1 where ts > case when ts then ts end; +if $rows != 0 then + return -1 +endi + +sql select sum(f1),count(f1) from tba1 partition by case when f1 then f1 when 1 then 1 end; +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != 5 then + return -1 +endi +if $data11 != 1 then + return -1 +endi + +sql select case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end caseWhen, sum(f1),count(f1) from tba1 group by case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end order by caseWhen; +if $rows != 3 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data02 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data11 != 5 then + return -1 +endi +if $data12 != 1 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data22 != 0 then + return -1 +endi + +sql select f1 from tba1 order by case when f1 <= 0 then 3 when f1 = 1 then 4 when f1 >= 3 then 2 else 1 end desc; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select cast(case f1 when f1 then f1 + 1 else f1 is null end as double) from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1.000000000 then + return -1 +endi +if $data10 != 2.000000000 then + return -1 +endi +if $data20 != 6.000000000 then + return -1 +endi +if $data30 != 1.000000000 then + return -1 +endi + +sql select sum(case f1 when f1 then f1 + 1 else f1 is null end + 1) from tba1; +if $rows != 1 then + return -1 +endi +if $data00 != 14.000000000 then + return -1 +endi + +sql select case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end,sum(f1),count(f1) from tba1 state_window(case when f1 < 3 then 1 when f1 >= 3 then 2 else 3 end); +if $rows != 3 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data02 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data11 != 5 then + return -1 +endi +if $data12 != 1 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data22 != 0 then + return -1 +endi + +sql select f1 from tba1 where case when case when f1 <= 0 then 3 when f1 = 1 then 4 when f1 >= 3 then 2 else 1 end > 2 then 1 else 0 end > 0; +if $rows != 2 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +sql select case when f1 is not null then case when f1 <= 0 then f1 else f1 * 10 end else -1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 10 then + return -1 +endi +if $data20 != 50 then + return -1 +endi +if $data30 != -1 then + return -1 +endi + + +print ======== case xx when xx + +sql select case 3 when 3 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 4 then + return -1 +endi +if $data20 != 4 then + return -1 +endi +if $data30 != 4 then + return -1 +endi + +sql select case 3 when 1 then 4 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case 3 when 1 then 4 else 2 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data20 != 2 then + return -1 +endi +if $data30 != 2 then + return -1 +endi + +sql select case 3 when null then 4 when '3' then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case '3' when null then 4 when 3 then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case null when null then 4 when 3 then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case 3.0 when null then 4 when '3' then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +sql select case f2 when 'a' then 4 when '0' then 1 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 4 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi + +sql select case f2 when f1 then f1 when f1 - 1 then f1 else 99 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != 99 then + return -1 +endi + +sql select case cast(f2 as int) when 0 then f2 when f1 then 11 else ts end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != a then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data20 != 11 then + return -1 +endi +if $data30 != 1664176504 then + return -1 +endi + +sql select case f1 + 1 when 1 then 1 when 2 then 2 else 3 end from tba1; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 2 then + return -1 +endi +if $data20 != 3 then + return -1 +endi +if $data30 != 3 then + return -1 +endi + +sql select case f1 when sum(f1) then sum(f1)-abs(f1) end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != 0.000000000 then + return -1 +endi +if $data20 != 0.000000000 then + return -1 +endi +if $data30 != 0.000000000 then + return -1 +endi + +sql select f1, case sum(f1) when 1 then f1 + 99 when f1 then f1 -99 else f1 end from tba1 group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != 0 then + return -1 +endi +if $data11 != -99.000000000 then + return -1 +endi +if $data20 != 1 then + return -1 +endi +if $data21 != 100.000000000 then + return -1 +endi +if $data30 != 5 then + return -1 +endi +if $data31 != -94.000000000 then + return -1 +endi + +sql select case when 3 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when null then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 1 then 4+1 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 1-1 then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 1+1 then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when abs(3) then abs(-1) end from sta; +if $rows != 20 then + return -1 +endi +sql select case when abs(1+1) then abs(-1)+abs(3) end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 1 else 3 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 1 when 1 then 0 else 3 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 1 when 1 then 0 when 2 then 3 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 'a' then 'b' when null then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when '2' then 'b' when null then 0 end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 'b' else null end from sta; +if $rows != 20 then + return -1 +endi +sql select case when 0 then 'b' else 2+abs(-2) end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when 3 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when 1 then 4 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when 1 then 4 else 2 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3 when null then 4 when '3' then 1 end from sta; +if $rows != 20 then + return -1 +endi +sql select case null when null then 4 when 3 then 1 end from sta; +if $rows != 20 then + return -1 +endi +sql select case 3.0 when null then 4 when '3' then 1 end from sta; +if $rows != 20 then + return -1 +endi +sql select f2,case f2 when 'a' then 4 when '0' then 1 end from sta order by f2; +if $rows != 20 then + return -1 +endi +sql select f2,f1,case f2 when f1 then f1 when f1 - 1 then f1 else 99 end from sta order by f2; +if $rows != 20 then + return -1 +endi +sql select case cast(f2 as int) when 0 then f2 when f1 then 11 else ts end from sta; +if $rows != 20 then + return -1 +endi +sql select f1, case f1 + 1 when 1 then 1 when 2 then 2 else 3 end from sta order by f1; +if $rows != 20 then + return -1 +endi +sql select case f1 when sum(f1) then sum(f1)-abs(f1) end from sta group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != 0.000000000 then + return -1 +endi +if $data20 != NULL then + return -1 +endi +if $data30 != NULL then + return -1 +endi +sql select case sum(f1) when 1 then f1 + 99 when f1 then f1 -99 else f1 end from sta group by f1 order by f1; +if $rows != 4 then + return -1 +endi +if $data00 != NULL then + return -1 +endi +if $data10 != -99.000000000 then + return -1 +endi +if $data20 != 1.000000000 then + return -1 +endi +if $data30 != 5.000000000 then + return -1 +endi + +sql select distinct tbname, case t1 when t2 then t1 else t1 + 100 end from sta order by tbname; +if $rows != 5 then + return -1 +endi +if $data01 != 0 then + return -1 +endi +if $data11 != 1 then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data31 != 101 then + return -1 +endi +if $data41 != 103 then + return -1 +endi + +sql_error select case when sum(f1) then sum(f1)-abs(f1) end from tba1; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 6fa7e393351b9378427e27f9695704ec8ad72006..252ef10b1a1c2813f073fe82590ee461b411361a 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -62,7 +62,16 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -v print =============== stepa: query data -sql select * from c1 + +$x = 0 +steps: + $x = $x + 1 + sleep 500 + if $x == 50 then + return -1 + endi +sql select * from c1 -x steps + sql select * from stb sql select * from stb_1 sql select ts, c1, c2, c3 from c1 diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 11a387ed4dffd7258120154adb9d7874ee0b21b5..89b6cadef88eb5a7d363b5fd6c784b7606095e66 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -158,9 +158,15 @@ print =============== restart system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -v -sleep 1000 +$x = 0 +steps: + $x = $x + 1 + sleep 500 + if $x == 50 then + return -1 + endi +sql select avg(tbcol) as c from stb -x steps -sql select avg(tbcol) as c from stb sql select avg(tbcol) as c from stb where ts <= 1601481840000 sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000 sql select avg(tbcol) as c from stb interval(1m) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py new file mode 100644 index 0000000000000000000000000000000000000000..d3a55ee0a6d97c23097c442321a079836c78d49f --- /dev/null +++ b/tests/system-test/1-insert/alter_database.py @@ -0,0 +1,58 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.buffer_boundary = [3,4097,8193,12289,16384] + self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] + # pages_boundary >= 64 + self.pages_boundary = [64,128,512] + self.pages_error = [self.pages_boundary[0]-1] + def alter_buffer(self): + tdSql.execute('create database db') + for buffer in self.buffer_boundary: + tdSql.execute(f'alter database db buffer {buffer}') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][8],buffer) + tdSql.execute('drop database db') + tdSql.execute('create database db vgroups 10') + for buffer in self.buffer_error: + tdSql.error(f'alter database db buffer {buffer}') + tdSql.execute('drop database db') + + def alter_pages(self): + tdSql.execute('create database db') + for pages in self.pages_boundary: + tdSql.execute(f'alter database db pages {pages}') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][10],pages) + tdSql.execute('drop database db') + tdSql.execute('create database db') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + self.pages_error.append(tdSql.queryResult[0][10]) + for pages in self.pages_error: + tdSql.error(f'alter database db pages {pages}') + tdSql.execute('drop database db') + + def run(self): + tdSql.error('create database db1 vgroups 10 buffer 12289') + self.alter_buffer() + self.alter_pages() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py new file mode 100644 index 0000000000000000000000000000000000000000..d18cb0260e90c721c37202e5d08bbaaa0a48da10 --- /dev/null +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -0,0 +1,77 @@ +import taos +import sys +import time +import socket +import os +import threading +from util.common import * + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.setsql = TDSetSql() + self.rowNum = 10 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + } + self.error_topic = ['avg','count','spread','stddev','sum','hyperloglog'] + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + def wrong_topic(self): + tdSql.prepare() + tdSql.execute('use db') + stbname = f'db.{tdCom.getLongName(5, "letters")}' + tag_dict = { + 't0':'int' + } + tag_values = [ + f'1' + ] + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + tdSql.execute(f"create table {stbname}_tb1 using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_tb1',self.rowNum) + for column in self.column_dict.keys(): + for func in self.error_topic: + if func.lower() != 'count' and column.lower() != 'ts': + tdSql.error(f'create topic tpn as select {func}({column}) from {stbname}') + elif func.lower() == 'count' : + tdSql.error(f'create topic tpn as select {func}(*) from {stbname}') + for column in self.column_dict.keys(): + if column.lower() != 'ts': + tdSql.error(f'create topic tpn as select apercentile({column},50) from {stbname}') + tdSql.error(f'create topic tpn as select leastquares({column},1,1) from {stbname}_tb1') + tdSql.error(f'create topic tpn as select HISTOGRAM({column},user_input,[1,3,5,7],0) from {stbname}') + tdSql.error(f'create topic tpn as select percentile({column},1) from {stbname}_tb1') + pass + def run(self): + self.wrong_topic() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index a613f11267ef769ef9162543a5818a0881f23abd..de95fef148aa29316e0a0a967ddf7e2acc787905 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -52,7 +52,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1,wal_retention_period=-1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1c6aef6286c5c1abeab1a4662741dfbc0ebf8961..39adfc9d530fdc7447135329aa033ad0e3869bf0 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,7 +2,7 @@ set -e set -x -#python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 0-others/telemetry.py @@ -18,7 +18,7 @@ python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py python3 ./test.py -f 0-others/compatibility.py - +python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py @@ -232,7 +232,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 -# unstable python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 5 -M 3 @@ -248,8 +248,8 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 - python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 -# python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 @@ -279,7 +279,7 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 - +python3 ./test.py -f 7-tmq/create_wrong_topic.py python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py @@ -302,7 +302,7 @@ python3 ./test.py -f 7-tmq/tmqCheckData.py python3 ./test.py -f 7-tmq/tmqCheckData1.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -python3 ./test.py -f 7-tmq/tmqShow.py +#python3 ./test.py -f 7-tmq/tmqShow.py python3 ./test.py -f 7-tmq/tmqAlterSchema.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py diff --git a/tools/scripts/codeFormat.sh b/tools/scripts/codeFormat.sh old mode 100644 new mode 100755 index 8707dcdabcfb1d015b6b3861b6734ca31a38f50f..cc7db5c723efcac9182956dda43c50aae4494e4f --- a/tools/scripts/codeFormat.sh +++ b/tools/scripts/codeFormat.sh @@ -12,25 +12,7 @@ FORMAT_DIR_LIST=( "${PRJ_ROOT_DIR}/source/os" "${PRJ_ROOT_DIR}/source/util" "${PRJ_ROOT_DIR}/source/common" - "${PRJ_ROOT_DIR}/source/libs/cache" - "${PRJ_ROOT_DIR}/source/libs/catalog" - "${PRJ_ROOT_DIR}/source/libs/command" - "${PRJ_ROOT_DIR}/source/libs/executor" - "${PRJ_ROOT_DIR}/source/libs/function" - "${PRJ_ROOT_DIR}/source/libs/index" - "${PRJ_ROOT_DIR}/source/libs/monitor" - "${PRJ_ROOT_DIR}/source/libs/nodes" - # "${PRJ_ROOT_DIR}/source/libs/parser" - "${PRJ_ROOT_DIR}/source/libs/planner" - "${PRJ_ROOT_DIR}/source/libs/qcom" - "${PRJ_ROOT_DIR}/source/libs/qworker" - "${PRJ_ROOT_DIR}/source/libs/scalar" - "${PRJ_ROOT_DIR}/source/libs/stream" - "${PRJ_ROOT_DIR}/source/libs/sync" - "${PRJ_ROOT_DIR}/source/libs/tdb" - "${PRJ_ROOT_DIR}/source/libs/tfs" - "${PRJ_ROOT_DIR}/source/libs/transport" - "${PRJ_ROOT_DIR}/source/libs/wal" + "${PRJ_ROOT_DIR}/source/libs" "${PRJ_ROOT_DIR}/source/client/inc" "${PRJ_ROOT_DIR}/source/client/src" "${PRJ_ROOT_DIR}/source/client/test" @@ -45,7 +27,7 @@ EXCLUDE_FILE_LIST=( ) for d in ${FORMAT_DIR_LIST[@]}; do - for f in $(find $d -type f -regex '.*\.\(cpp\|hpp\|c\|h\)'); do + for f in $(find $d -type f -not -name '*sql.c' -regex '.*\.\(cpp\|hpp\|c\|h\)'); do ${FORMAT_BIN} -i $f done done diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 8aafd747afe2f5972e0010d0be0222a7d56e961b..46eaf6cff1a7d070a88ae4fa1dc67cb0512de75a 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -325,24 +325,23 @@ int cntDel = 0; // delete byte count after next press tab // show auto tab introduction void printfIntroduction() { - printf(" **************************** How To Use TAB Key ********************************\n"); - printf(" * TDengine Command Line supports pressing TAB key to complete word, *\n"); - printf(" * including database name, table name, function name and keywords. *\n"); - printf(" * Press TAB key anywhere, you'll get surprise. *\n"); - printf(" * KEYBOARD SHORTCUT: *\n"); - printf(" * [ TAB ] ...... Complete the word or show help if no input *\n"); - printf(" * [ Ctrl + A ] ...... move cursor to [A]head of line *\n"); - printf(" * [ Ctrl + E ] ...... move cursor to [E]nd of line *\n"); - printf(" * [ Ctrl + W ] ...... move cursor to line of middle *\n"); - printf(" * [ Ctrl + L ] ...... clean screen *\n"); - printf(" * [ Ctrl + K ] ...... clean after cursor *\n"); - printf(" * [ Ctrl + U ] ...... clean before cursor *\n"); - printf(" * *\n"); + printf(" ****************************** Tab Completion **********************************\n"); + printf(" * The TDengine CLI supports tab completion for a variety of items, *\n"); + printf(" * including database names, table names, function names and keywords. *\n"); + printf(" * The full list of shortcut keys is as follows: *\n"); + printf(" * [ TAB ] ...... complete the current word *\n"); + printf(" * ...... if used on a blank line, display all valid commands *\n"); + printf(" * [ Ctrl + A ] ...... move cursor to the st[A]rt of the line *\n"); + printf(" * [ Ctrl + E ] ...... move cursor to the [E]nd of the line *\n"); + printf(" * [ Ctrl + W ] ...... move cursor to the middle of the line *\n"); + printf(" * [ Ctrl + L ] ...... clear the entire screen *\n"); + printf(" * [ Ctrl + K ] ...... clear the screen after the cursor *\n"); + printf(" * [ Ctrl + U ] ...... clear the screen before the cursor *\n"); printf(" **********************************************************************************\n\n"); } void showHelp() { - printf("\nThe following are supported commands for TDengine Command Line:"); + printf("\nThe TDengine CLI supports the following commands:"); printf( "\n\ ----- A ----- \n\ diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index fd0ea6032357db68fc523a61f56de6aa0df54017..37d00c0a7ce9d5a2d91ae7a2f9747097361e2c1f 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -229,7 +229,7 @@ void shellRunSingleCommandImp(char *command) { printf("Query OK, %d of %d rows affected (%.6fs)\r\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); // call auto tab - callbackAutoTab(command, pSql, false); + callbackAutoTab(command, NULL, false); } printf("\r\n"); @@ -998,7 +998,9 @@ void *shellCancelHandler(void *arg) { shell.stop_query = true; } else { #endif - taos_kill_query(shell.conn); + if (shell.conn) { + taos_kill_query(shell.conn); + } #ifdef WEBSOCKET } #endif diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c index 2f1ee12d5414dd31891186fad2e7192206947973..b3102811197adc7044bf9d4682b05c795b9632d6 100644 --- a/tools/shell/src/shellTire.c +++ b/tools/shell/src/shellTire.c @@ -243,8 +243,8 @@ void enumAllWords(STireNode** nodes, char* prefix, SMatch* match) { continue; } else { // combine word string - memset(word, 0, sizeof(word)); - strcpy(word, prefix); + memset(word, 0, tListLen(word)); + strncpy(word, prefix, len); word[len] = FIRST_ASCII + i; // append current char // chain middle node @@ -315,8 +315,7 @@ void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) { } } - // return - return; + taosMemoryFree(root); } SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match) { diff --git a/tools/taosws-rs b/tools/taosws-rs new file mode 160000 index 0000000000000000000000000000000000000000..7a94ffab45f08e16f09b3f430fe75d717054adb6 --- /dev/null +++ b/tools/taosws-rs @@ -0,0 +1 @@ +Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6 diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 56ba622a9cc522cd77b1126f0efe916ed5f4a0e6..b6d8d75ba01c8930787f8fad434348ce9ee571d1 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -119,7 +119,7 @@ int smlProcess_json1_Test() { " \"dc\": \"lga\"" " }" " }" - "]"}; + "]",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -159,7 +159,7 @@ int smlProcess_json2_Test() { " }," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -227,7 +227,7 @@ int smlProcess_json3_Test() { " }," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -286,7 +286,7 @@ int smlProcess_json4_Test() { " \"t9\": false," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index a043aa7a6d75effa224d77d1de518a35abd06249..d36fe0855aa92fa4e052526d558faf12bf4e78c7 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -155,7 +155,7 @@ static void printHelp() { printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); + printf("%s%s%s%d\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "producer thread number, default is 0"); printf("%s%s\n", indent, "-b"); @@ -238,7 +238,7 @@ void saveConfigToLogFile() { taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); } taosFprintfFile(g_fp, "\n"); - taosFprintfFile(g_fp, " expect rows: %d\n", g_stConfInfo.stThreads[i].expectMsgCnt); + taosFprintfFile(g_fp, " expect rows: %" PRIx64 "\n", g_stConfInfo.stThreads[i].expectMsgCnt); } char tmpString[128]; @@ -263,11 +263,11 @@ void parseArgument(int32_t argc, char* argv[]) { printHelp(); exit(0); } else if (strcmp(argv[i], "-d") == 0) { - strcpy(g_stConfInfo.dbName, argv[++i]); + tstrncpy(g_stConfInfo.dbName, argv[++i], sizeof(g_stConfInfo.dbName)); } else if (strcmp(argv[i], "-w") == 0) { - strcpy(g_stConfInfo.cdbName, argv[++i]); + tstrncpy(g_stConfInfo.cdbName, argv[++i], sizeof(g_stConfInfo.cdbName)); } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); + tstrncpy(configDir, argv[++i], PATH_MAX); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { @@ -279,9 +279,9 @@ void parseArgument(int32_t argc, char* argv[]) { } else if (strcmp(argv[i], "-e") == 0) { g_stConfInfo.useSnapshot = atol(argv[++i]); } else if (strcmp(argv[i], "-t") == 0) { - char tmpBuf[56]; - strcpy(tmpBuf, argv[++i]); - sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); + char tmpBuf[56] = {0}; + tstrncpy(tmpBuf, argv[++i], sizeof(tmpBuf)); + sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); } else if (strcmp(argv[i], "-x") == 0) { g_stConfInfo.numOfThread = atol(argv[++i]); } else if (strcmp(argv[i], "-l") == 0) { @@ -294,6 +294,10 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.producerRate = atol(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { g_stConfInfo.payloadLen = atol(argv[++i]); + if(g_stConfInfo.payloadLen <= 0 || g_stConfInfo.payloadLen > 1024 * 1024 * 1024){ + pError("%s calloc size is too large: %s %s", GREEN, argv[++i], NC); + exit(-1); + } } else { pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); @@ -354,8 +358,8 @@ void ltrim(char* str) { int queryDB(TAOS* taos, char* command) { int retryCnt = 10; - int code; - TAOS_RES* pRes; + int code = 0; + TAOS_RES* pRes = NULL; while (retryCnt--) { pRes = taos_query(taos, command); @@ -363,10 +367,11 @@ int queryDB(TAOS* taos, char* command) { if (code != 0) { taosSsleep(1); taos_free_result(pRes); + pRes = NULL; continue; } taos_free_result(pRes); - return 0; + return 0; } pError("failed to reason:%s, sql: %s", tstrerror(code), command); @@ -418,7 +423,7 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { char sqlStr[1100] = {0}; if (strlen(buf) > 1024) { - taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); + taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", (int)strlen(buf)); taosCloseFile(&g_fp); return -1; } @@ -592,7 +597,7 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId); @@ -644,7 +649,7 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId); @@ -960,7 +965,7 @@ void parseConsumeInfo() { ltrim(pstr); char* ret = strchr(pstr, ch); memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); - strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); + tstrncpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1, sizeof(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey])); // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], // g_stConfInfo.value[g_stConfInfo.numOfKey]); g_stConfInfo.stThreads[i].numOfKey++; @@ -1268,25 +1273,26 @@ void* ombProduceThreadFunc(void* param) { for (int i = 0; i < batchPerTblTimes; ++i) { uint32_t msgsOfSql = g_stConfInfo.batchSize; if ((i == batchPerTblTimes - 1) && (0 != remainder)) { - msgsOfSql = remainder; + msgsOfSql = remainder; } int len = 0; len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName); for (int j = 0; j < msgsOfSql; j++) { - int64_t timeStamp = taosGetTimestampNs(); - len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); + int64_t timeStamp = taosGetTimestampNs(); + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); sendMsgs++; pInfo->totalProduceMsgs++; } - totalMsgLen += len; + totalMsgLen += len; pInfo->totalMsgsLen += len; - int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); + int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); if (affectedRows < 0) { taos_close(pInfo->taos); - pInfo->taos = NULL; - return NULL; + pInfo->taos = NULL; + taosMemoryFree(sqlBuf); + return NULL; } affectedRowsTotal += affectedRows; @@ -1322,6 +1328,7 @@ void* ombProduceThreadFunc(void* param) { printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); taos_close(pInfo->taos); pInfo->taos = NULL; + taosMemoryFree(sqlBuf); return NULL; } diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 3a598ba98b59d4e55f1a12cc7dd60745ac915d74..6540fdac4cdba1aa9fe422719f91b13ce365c7d9 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -663,7 +663,7 @@ void initLogFile() { int main(int argc, char* argv[]) { for (int32_t i = 1; i < argc; i++) { if(strcmp(argv[i], "-c") == 0){ - strcpy(g_conf.dir, argv[++i]); + tstrncpy(g_conf.dir, argv[++i], sizeof(g_conf.dir)); }else if(strcmp(argv[i], "-s") == 0){ g_conf.snapShot = true; }else if(strcmp(argv[i], "-d") == 0){