diff --git a/cmake/cmake.define b/cmake/cmake.define index 55412fd26f8749eabdd0451ed15d030932754edd..1d34896f9aa1636f493f1d844f2e4d34daff8077 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -71,8 +71,8 @@ ELSE () ENDIF () IF (${SANITIZER} MATCHES "true") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") MESSAGE(STATUS "Will compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") diff --git a/cmake/cmake.options b/cmake/cmake.options index c77b580c17e6d7c7f32ffa24fddd01ecb0f1e394..cb6fd1400d43b6073d81ab43e46140343b277512 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -146,5 +146,6 @@ option( option( BUILD_WITH_INVERTEDINDEX "If use invertedIndex" - ON + OFF ) + diff --git a/docs-cn/07-develop/01-connect/index.md b/docs-cn/07-develop/01-connect/index.md index ebdefc77b9cc23712626f7543e0e5cc29db3e080..3a15d03f93cee7dd064f29b4911019cae3632b9a 100644 --- a/docs-cn/07-develop/01-connect/index.md +++ b/docs-cn/07-develop/01-connect/index.md @@ -33,7 +33,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 关键不同点在于： 1. 使用 REST 连接，用户无需安装客户端驱动程序 taosc，具有跨平台易用的优势，但性能要下降 30%左右。 -2. 使用原生连接可以体验 TDengine 的全部功能，如[参数绑定接口](/reference/connector/cpp#参数绑定-api)、[订阅](reference/connector/cpp#数据订阅接口)等等。 +2. 使用原生连接可以体验 TDengine 的全部功能，如[参数绑定接口](/reference/connector/cpp#参数绑定-api)、[订阅](/reference/connector/cpp#订阅和消费-api)等等。 ## 安装客户端驱动 taosc diff --git a/docs-cn/12-taos-sql/02-database.md b/docs-cn/12-taos-sql/02-database.md index 1454d1d34415ea5af9b019ae6eea36923e6d05be..566fec324148fede8d897869656b83e657569f59 100644 --- a/docs-cn/12-taos-sql/02-database.md +++ b/docs-cn/12-taos-sql/02-database.md @@ -20,21 +20,21 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1]; 3. 数据库名最大长度为 33； 4. 一条 SQL 语句的最大长度为 65480 个字符； 5. 创建数据库时可用的参数有： - - cache: [Description](/reference/config/#cache) - - blocks: [Description](/reference/config/#blocks) - - days: [Description](/reference/config/#days) - - keep: [Description](/reference/config/#keep) - - minRows: [Description](/reference/config/#minrows) - - maxRows: [Description](/reference/config/#maxrows) - - wal: [Description](/reference/config/#wallevel) - - fsync: [Description](/reference/config/#fsync) - - update: [Description](/reference/config/#update) - - cacheLast: [Description](/reference/config/#cachelast) - - replica: [Description](/reference/config/#replica) - - quorum: [Description](/reference/config/#quorum) - - maxVgroupsPerDb: [Description](/reference/config/#maxvgroupsperdb) - - comp: [Description](/reference/config/#comp) - - precision: [Description](/reference/config/#precision) + - cache: [详细说明](/reference/config/#cache) + - blocks: [详细说明](/reference/config/#blocks) + - days: [详细说明](/reference/config/#days) + - keep: [详细说明](/reference/config/#keep) + - minRows: [详细说明](/reference/config/#minrows) + - maxRows: [详细说明](/reference/config/#maxrows) + - wal: [详细说明](/reference/config/#wallevel) + - fsync: [详细说明](/reference/config/#fsync) + - update: [详细说明](/reference/config/#update) + - cacheLast: [详细说明](/reference/config/#cachelast) + - replica: [详细说明](/reference/config/#replica) + - quorum: [详细说明](/reference/config/#quorum) + - maxVgroupsPerDb: [详细说明](/reference/config/#maxvgroupsperdb) + - comp: [详细说明](/reference/config/#comp) + - precision: [详细说明](/reference/config/#precision) 6. 请注意上面列出的所有参数都可以配置在配置文件 `taosd.cfg` 中作为创建数据库时使用的默认配置， `create database` 的参数中明确指定的会覆盖配置文件中的设置。 ::: diff --git a/docs-en/07-develop/01-connect/index.md b/docs-en/07-develop/01-connect/index.md index ecb8caa308da147adc191b98af9df81c7af1eb0b..ee11c8f5445233abe44e3bc006e1f15846b54ada 100644 --- a/docs-en/07-develop/01-connect/index.md +++ b/docs-en/07-develop/01-connect/index.md @@ -33,7 +33,7 @@ Either way, same or similar APIs are provided by connectors to access database o Key differences： 1. With REST connection, it's not necessary to install TDengine client driver (taosc), it's more friendly for cross-platform with the cost of 30% performance downgrade. When taosc has an upgrade, application does not need to make changes. -2. With native connection, full compatibility of TDengine can be utilized, like [Parameter Binding](/reference/connector/cpp#Parameter Binding-api), [Subscription](reference/connector/cpp#Subscription), etc. But taosc has to be installed, some platforms may not be supported. +2. With native connection, full compatibility of TDengine can be utilized, like [Parameter Binding](/reference/connector/cpp#parameter-binding-api), [Subscription](/reference/connector/cpp#subscription-and-consumption-api), etc. But taosc has to be installed, some platforms may not be supported. ## Install Client Driver taosc diff --git a/docs-en/12-taos-sql/02-database.md b/docs-en/12-taos-sql/02-database.md index 12e2edf8bae21059e8c2d5c18858d502c834e9c1..85b71bbde727ea1ff84080d3770e641d59b88c7b 100644 --- a/docs-en/12-taos-sql/02-database.md +++ b/docs-en/12-taos-sql/02-database.md @@ -34,7 +34,7 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1]; - quorum: [Description](/reference/config/#quorum) - maxVgroupsPerDb: [Description](/reference/config/#maxvgroupsperdb) - comp: [Description](/reference/config/#comp) - - precision: [Description](reference/config/#precision) + - precision: [Description](/reference/config/#precision) 6. Please be noted that all of the parameters mentioned in this section can be configured in configuration file `taosd.cfg` at server side and used by default, can be override if they are specified in `create database` statement. ::: diff --git a/docs-en/14-reference/03-connector/node.mdx b/docs-en/14-reference/03-connector/node.mdx index 48f724426a96e62e5b56ab4285e5c5fabc95c765..3d30148e8ed9d8f98d135fa0fa72809f1115231a 100644 --- a/docs-en/14-reference/03-connector/node.mdx +++ b/docs-en/14-reference/03-connector/node.mdx @@ -78,7 +78,7 @@ Manually install the following tools. - Install [Python](https://www.python.org/downloads/) 2.7 (`v3.x.x` is not supported) and execute `npm config set python python2.7`. - Go to the `cmd` command-line interface, `npm config set msvs_version 2017` -Refer to Microsoft's Node.js User Manual [Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows- environment. md#compiling-native-addon-modules). +Refer to Microsoft's Node.js User Manual [Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules). If using ARM64 Node.js on Windows 10 ARM, you must add "Visual C++ compilers and libraries for ARM64" and "Visual C++ ATL for ARM64". diff --git a/docs-en/25-application/03-immigrate.md b/docs-en/25-application/03-immigrate.md index 81d5f512bfc8ed3cd1f5223a9ff72218023515f0..4cfeb892d821a1e5b7d5250615e7122e64b9882d 100644 --- a/docs-en/25-application/03-immigrate.md +++ b/docs-en/25-application/03-immigrate.md @@ -32,7 +32,7 @@ We will explain how to migrate OpenTSDB applications to TDengine quickly, secure The following figure (Figure 1) shows the system's overall architecture for a typical DevOps application scenario. **Figure 1. Typical architecture in a DevOps scenario** -Figure 1. [IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch](/img/IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.jpg "Figure 1. Typical architecture in a DevOps scenario") +![IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch](/img/IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.jpg "Figure 1. Typical architecture in a DevOps scenario") In this application scenario, there are Agent tools deployed in the application environment to collect machine metrics, network metrics, and application metrics. Data collectors to aggregate information collected by agents, systems for persistent data storage and management, and tools for monitoring data visualization (e.g., Grafana, etc.). @@ -75,7 +75,7 @@ After writing the data to TDengine properly, you can adapt Grafana to visualize TDengine provides two sets of Dashboard templates by default, and users only need to import the templates from the Grafana directory into Grafana to activate their use. **Importing Grafana Templates** Figure 2. -! [](/img/IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.jpg "Figure 2. Importing a Grafana Template") +![](/img/IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.jpg "Figure 2. Importing a Grafana Template") After the above steps, you completed the migration to replace OpenTSDB with TDengine. You can see that the whole process is straightforward, there is no need to write any code, and only some configuration files need to be adjusted to meet the migration work. @@ -88,7 +88,7 @@ In most DevOps scenarios, if you have a small OpenTSDB cluster (3 or fewer nodes Suppose your application is particularly complex, or the application domain is not a DevOps scenario. You can continue reading subsequent chapters for a more comprehensive and in-depth look at the advanced topics of migrating an OpenTSDB application to TDengine. **Figure 3. System architecture after migration** -! [IT-DevOps-Solutions-Immigrate-TDengine-Arch](/img/IT-DevOps-Solutions-Immigrate-TDengine-Arch.jpg "Figure 3. System architecture after migration completion") +![IT-DevOps-Solutions-Immigrate-TDengine-Arch](/img/IT-DevOps-Solutions-Immigrate-TDengine-Arch.jpg "Figure 3. System architecture after migration completion") ## Migration evaluation and strategy for other scenarios @@ -96,7 +96,7 @@ Suppose your application is particularly complex, or the application domain is n This chapter describes the differences between OpenTSDB and TDengine at the system functionality level. After reading this chapter, you can fully evaluate whether you can migrate some complex OpenTSDB-based applications to TDengine, and what you should pay attention to after migration. -TDengine currently only supports Grafana for visual kanban rendering, so if your application uses front-end kanban boards other than Grafana (e.g., [TSDash](https://github.com/facebook/tsdash), [Status Wolf](https://github) .com/box/StatusWolf), etc.). You cannot directly migrate those front-end kanbans to TDengine, and the front-end kanban will need to be ported to Grafana to work correctly. +TDengine currently only supports Grafana for visual kanban rendering, so if your application uses front-end kanban boards other than Grafana (e.g., [TSDash](https://github.com/facebook/tsdash), [Status Wolf](https://github.com/box/StatusWolf), etc.). You cannot directly migrate those front-end kanbans to TDengine, and the front-end kanban will need to be ported to Grafana to work correctly. TDengine version 2.3.0.x only supports collectd and StatsD as data collection aggregation software but will provide more data collection aggregation software in the future. If you use other data aggregators on the collection side, your application needs to be ported to these two data aggregation systems to write data correctly. In addition to the two data aggregator software protocols mentioned above, TDengine also supports writing data directly via InfluxDB's line protocol and OpenTSDB's data writing protocol, JSON format. You can rewrite the logic on the data push side to write data using the line protocols supported by TDengine. diff --git a/docs-examples/.gitignore b/docs-examples/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..7ed6d403bf5f64c0cb230265b4dffee609dea93b --- /dev/null +++ b/docs-examples/.gitignore @@ -0,0 +1,3 @@ +.vscode +*.lock +.idea \ No newline at end of file diff --git a/docs-examples/.gitignre b/docs-examples/.gitignre new file mode 100644 index 0000000000000000000000000000000000000000..0853156c65c2c6c1b693290e74c3ee630bcaac19 --- /dev/null +++ b/docs-examples/.gitignre @@ -0,0 +1,2 @@ +.vscode +*.lock \ No newline at end of file diff --git a/docs-examples/R/connect_native.r b/docs-examples/R/connect_native.r new file mode 100644 index 0000000000000000000000000000000000000000..18c142872be5efaa7167c10a25f62bcb9fbf5a52 --- /dev/null +++ b/docs-examples/R/connect_native.r @@ -0,0 +1,16 @@ +if (! "RJDBC" %in% installed.packages()[, "Package"]) { + install.packages('RJDBC', repos='http://cran.us.r-project.org') +} + +# ANCHOR: demo +library("DBI") +library("rJava") +library("RJDBC") + +args<- commandArgs(trailingOnly = TRUE) +driver_path = args[1] # path to jdbc-driver for example: "/root/taos-jdbcdriver-2.0.37-dist.jar" +driver = JDBC("com.taosdata.jdbc.TSDBDriver", driver_path) +conn = dbConnect(driver, "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata") +dbGetQuery(conn, "SELECT server_version()") +dbDisconnect(conn) +# ANCHOR_END: demo \ No newline at end of file diff --git a/docs-examples/R/connect_rest.r b/docs-examples/R/connect_rest.r new file mode 100644 index 0000000000000000000000000000000000000000..5ceec572fc26575dfc597983eeac3233bc4488ab --- /dev/null +++ b/docs-examples/R/connect_rest.r @@ -0,0 +1,12 @@ +if (! "RJDBC" %in% installed.packages()[, "Package"]) { + install.packages('RJDBC', repos='http://cran.us.r-project.org') +} + +library("DBI") +library("rJava") +library("RJDBC") +driver_path = "/home/debug/build/lib/taos-jdbcdriver-2.0.38-dist.jar" +driver = JDBC("com.taosdata.jdbc.rs.RestfulDriver", driver_path) +conn = dbConnect(driver, "jdbc:TAOS-RS://localhost:6041?user=root&password=taosdata") +dbGetQuery(conn, "SELECT server_version()") +dbDisconnect(conn) \ No newline at end of file diff --git a/docs-examples/c/.gitignore b/docs-examples/c/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..afe974314989a1e3aa4eee703738a9a960c18577 --- /dev/null +++ b/docs-examples/c/.gitignore @@ -0,0 +1,3 @@ +* +!*.c +!.gitignore diff --git a/docs-examples/c/async_query_example.c b/docs-examples/c/async_query_example.c new file mode 100644 index 0000000000000000000000000000000000000000..262757f02b5c52f2d4402d363663db80bb38a54d --- /dev/null +++ b/docs-examples/c/async_query_example.c @@ -0,0 +1,195 @@ +// compile with: +// gcc -o async_query_example async_query_example.c -ltaos + +#include +#include +#include +#include +#include +#include + +typedef int16_t VarDataLenT; + +#define TSDB_NCHAR_SIZE sizeof(int32_t) +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + +#define GET_FLOAT_VAL(x) (*(float *)(x)) +#define GET_DOUBLE_VAL(x) (*(double *)(x)) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] + +int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) { + int len = 0; + char split = ' '; + + for (int i = 0; i < numFields; ++i) { + if (i > 0) { + str[len++] = split; + } + + if (row[i] == NULL) { + len += sprintf(str + len, "%s", "NULL"); + continue; + } + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(str + len, "%d", *((int16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_INT: + len += sprintf(str + len, "%d", *((int32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UINT: + len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_FLOAT: { + float fv = 0; + fv = GET_FLOAT_VAL(row[i]); + len += sprintf(str + len, "%f", fv); + } break; + + case TSDB_DATA_TYPE_DOUBLE: { + double dv = 0; + dv = GET_DOUBLE_VAL(row[i]); + len += sprintf(str + len, "%lf", dv); + } break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); + memcpy(str + len, row[i], charLen); + len += charLen; + } break; + + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BOOL: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + default: + break; + } + } + + return len; +} + +void printHeader(TAOS_RES *res) { + int numFields = taos_num_fields(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + char header[256] = {0}; + int len = 0; + for (int i = 0; i < numFields; ++i) { + len += sprintf(header + len, "%s ", fields[i].name); + } + puts(header); +} + +// ANCHOR: demo + +/** + * @brief call back function of taos_fetch_row_a + * + * @param param : the third parameter you passed to taos_fetch_row_a + * @param res : pointer of TAOS_RES + * @param numOfRow : number of rows fetched in this batch. will be 0 if there is no more data. + * @return void* + */ +void *fetch_row_callback(void *param, TAOS_RES *res, int numOfRow) { + printf("numOfRow = %d \n", numOfRow); + int numFields = taos_num_fields(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + TAOS *_taos = (TAOS *)param; + if (numOfRow > 0) { + for (int i = 0; i < numOfRow; ++i) { + TAOS_ROW row = taos_fetch_row(res); + char temp[256] = {0}; + printRow(temp, row, fields, numFields); + puts(temp); + } + taos_fetch_rows_a(res, fetch_row_callback, _taos); + } else { + printf("no more data, close the connection.\n"); + taos_free_result(res); + taos_close(_taos); + taos_cleanup(); + } +} + +/** + * @brief callback function of taos_query_a + * + * @param param: the fourth parameter you passed to taos_query_a + * @param res : the result set + * @param code : status code + * @return void* + */ +void *select_callback(void *param, TAOS_RES *res, int code) { + printf("query callback ...\n"); + TAOS *_taos = (TAOS *)param; + if (code == 0 && res) { + printHeader(res); + taos_fetch_rows_a(res, fetch_row_callback, _taos); + } else { + printf("failed to execute taos_query. error: %s\n", taos_errstr(res)); + taos_free_result(res); + taos_close(_taos); + taos_cleanup(); + exit(EXIT_FAILURE); + } +} + +int main() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", "power", 6030); + if (taos == NULL) { + puts("failed to connect to server"); + exit(EXIT_FAILURE); + } + // param one is the connection returned by taos_connect. + // param two is the SQL to execute. + // param three is the callback function. + // param four can be any pointer. It will be passed to your callback function as the first parameter. we use taos + // here, because we want to close it after getting data. + taos_query_a(taos, "SELECT * FROM meters", select_callback, taos); + sleep(1); +} + +// output: +// query callback ... +// ts current voltage phase location groupid +// numOfRow = 8 +// 1538548685000 10.300000 219 0.310000 beijing.chaoyang 2 +// 1538548695000 12.600000 218 0.330000 beijing.chaoyang 2 +// 1538548696800 12.300000 221 0.310000 beijing.chaoyang 2 +// 1538548696650 10.300000 218 0.250000 beijing.chaoyang 3 +// 1538548685500 11.800000 221 0.280000 beijing.haidian 2 +// 1538548696600 13.400000 223 0.290000 beijing.haidian 2 +// 1538548685000 10.800000 223 0.290000 beijing.haidian 3 +// 1538548686500 11.500000 221 0.350000 beijing.haidian 3 +// numOfRow = 0 +// no more data, close the connection. +// ANCHOR_END: demo \ No newline at end of file diff --git a/docs-examples/c/connect_example.c b/docs-examples/c/connect_example.c new file mode 100644 index 0000000000000000000000000000000000000000..1a23df4806d7ff986898734e1971f6e0cd7c5360 --- /dev/null +++ b/docs-examples/c/connect_example.c @@ -0,0 +1,24 @@ +// compile with +// gcc connect_example.c -o connect_example -ltaos +#include +#include +#include "taos.h" + +int main() { + const char *host = "localhost"; + const char *user = "root"; + const char *passwd = "taosdata"; + // if don't want to connect to a default db, set it to NULL or "" + const char *db = NULL; + uint16_t port = 0; // 0 means use the default port + TAOS *taos = taos_connect(host, user, passwd, db, port); + if (taos == NULL) { + int errno = taos_errno(NULL); + char *msg = taos_errstr(NULL); + printf("%d, %s\n", errno, msg); + } else { + printf("connected\n"); + taos_close(taos); + } + taos_cleanup(); +} diff --git a/docs-examples/c/error_handle_example.c b/docs-examples/c/error_handle_example.c new file mode 100644 index 0000000000000000000000000000000000000000..e7dedb263df250f6634aa15fab2729cbaf4e5972 --- /dev/null +++ b/docs-examples/c/error_handle_example.c @@ -0,0 +1,24 @@ +// compile with +// gcc error_handle_example.c -o error_handle_example -ltaos +#include +#include +#include "taos.h" + +int main() { + const char *host = "localhost"; + const char *user = "root"; + const char *passwd = "taosdata"; + // if don't want to connect to a default db, set it to NULL or "" + const char *db = "notexist"; + uint16_t port = 0; // 0 means use the default port + TAOS *taos = taos_connect(host, user, passwd, db, port); + if (taos == NULL) { + int errno = taos_errno(NULL); + char *msg = taos_errstr(NULL); + printf("%d, %s\n", errno, msg); + } else { + printf("connected\n"); + taos_close(taos); + } + taos_cleanup(); +} diff --git a/docs-examples/c/insert_example.c b/docs-examples/c/insert_example.c new file mode 100644 index 0000000000000000000000000000000000000000..ca12be9314efbda707dbd05449c746794c209743 --- /dev/null +++ b/docs-examples/c/insert_example.c @@ -0,0 +1,51 @@ +// compile with +// gcc -o insert_example insert_example.c -ltaos +#include +#include +#include "taos.h" + + +/** + * @brief execute sql and print affected rows. + * + * @param taos + * @param sql + */ +void executeSQL(TAOS *taos, const char *sql) { + TAOS_RES *res = taos_query(taos, sql); + int code = taos_errno(res); + if (code != 0) { + printf("Error code: %d; Message: %s\n", code, taos_errstr(res)); + taos_free_result(res); + taos_close(taos); + exit(EXIT_FAILURE); + } + int affectedRows = taos_affected_rows(res); + printf("affected rows %d\n", affectedRows); + taos_free_result(res); +} + + + +int main() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 6030); + if (taos == NULL) { + printf("failed to connect to server\n"); + exit(EXIT_FAILURE); + } + executeSQL(taos, "CREATE DATABASE power"); + executeSQL(taos, "USE power"); + executeSQL(taos, "CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)"); + executeSQL(taos, "INSERT INTO d1001 USING meters TAGS(Beijing.Chaoyang, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)" + "d1002 USING meters TAGS(Beijing.Chaoyang, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)" + "d1003 USING meters TAGS(Beijing.Haidian, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000)" + "d1004 USING meters TAGS(Beijing.Haidian, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"); + taos_close(taos); + taos_cleanup(); +} + +// output: +// affected rows 0 +// affected rows 0 +// affected rows 0 +// affected rows 8 \ No newline at end of file diff --git a/docs-examples/c/json_protocol_example.c b/docs-examples/c/json_protocol_example.c new file mode 100644 index 0000000000000000000000000000000000000000..182fd201308facc80c76f36cfa57580784d70413 --- /dev/null +++ b/docs-examples/c/json_protocol_example.c @@ -0,0 +1,52 @@ +// compile with +// gcc -o json_protocol_example json_protocol_example.c -ltaos +#include +#include +#include +#include "taos.h" + +void executeSQL(TAOS *taos, const char *sql) { + TAOS_RES *res = taos_query(taos, sql); + int code = taos_errno(res); + if (code != 0) { + printf("%s\n", taos_errstr(res)); + taos_free_result(res); + taos_close(taos); + exit(EXIT_FAILURE); + } + taos_free_result(res); +} + +// ANCHOR: main +int main() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", "", 6030); + if (taos == NULL) { + printf("failed to connect to server\n"); + exit(EXIT_FAILURE); + } + executeSQL(taos, "DROP DATABASE IF EXISTS test"); + executeSQL(taos, "CREATE DATABASE test"); + executeSQL(taos, "USE test"); + char *line = + "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": " + "\"Beijing.Chaoyang\", \"groupid\": 2}},{\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, " + "\"value\": 219, \"tags\": {\"location\": \"Beijing.Haidian\", \"groupid\": 1}},{\"metric\": \"meters.current\", " + "\"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"Beijing.Chaoyang\", \"groupid\": " + "2}},{\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": " + "\"Beijing.Haidian\", \"groupid\": 1}}]"; + + char *lines[] = {line}; + TAOS_RES *res = taos_schemaless_insert(taos, lines, 1, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + if (taos_errno(res) != 0) { + printf("failed to insert schema-less data, reason: %s\n", taos_errstr(res)); + } else { + int affectedRow = taos_affected_rows(res); + printf("successfully inserted %d rows\n", affectedRow); + } + taos_free_result(res); + taos_close(taos); + taos_cleanup(); +} +// output: +// successfully inserted 4 rows +// ANCHOR_END: main diff --git a/docs-examples/c/line_example.c b/docs-examples/c/line_example.c new file mode 100644 index 0000000000000000000000000000000000000000..8dd4b1a5075369625645959da0476b76b9fbf290 --- /dev/null +++ b/docs-examples/c/line_example.c @@ -0,0 +1,47 @@ +// compile with +// gcc -o line_example line_example.c -ltaos +#include +#include +#include +#include "taos.h" + +void executeSQL(TAOS *taos, const char *sql) { + TAOS_RES *res = taos_query(taos, sql); + int code = taos_errno(res); + if (code != 0) { + printf("%s\n", taos_errstr(res)); + taos_free_result(res); + taos_close(taos); + exit(EXIT_FAILURE); + } + taos_free_result(res); +} + +// ANCHOR: main +int main() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", "", 0); + if (taos == NULL) { + printf("failed to connect to server\n"); + exit(EXIT_FAILURE); + } + executeSQL(taos, "DROP DATABASE IF EXISTS test"); + executeSQL(taos, "CREATE DATABASE test"); + executeSQL(taos, "USE test"); + char *lines[] = {"meters,location=Beijing.Haidian,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249", + "meters,location=Beijing.Haidian,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250", + "meters,location=Beijing.Haidian,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249", + "meters,location=Beijing.Haidian,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250"}; + TAOS_RES *res = taos_schemaless_insert(taos, lines, 4, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + if (taos_errno(res) != 0) { + printf("failed to insert schema-less data, reason: %s\n", taos_errstr(res)); + } else { + int affectedRows = taos_affected_rows(res); + printf("successfully inserted %d rows\n", affectedRows); + } + taos_free_result(res); + taos_close(taos); + taos_cleanup(); +} +// output: +// successfully inserted 4 rows +// ANCHOR_END: main \ No newline at end of file diff --git a/docs-examples/c/multi_bind_example.c b/docs-examples/c/multi_bind_example.c new file mode 100644 index 0000000000000000000000000000000000000000..fe11df9caad3e216fbd0b1ff2f40a54fe3ba86e5 --- /dev/null +++ b/docs-examples/c/multi_bind_example.c @@ -0,0 +1,147 @@ +// compile with +// gcc -o multi_bind_example multi_bind_example.c -ltaos +#include +#include +#include +#include "taos.h" + +/** + * @brief execute sql only and ignore result set + * + * @param taos + * @param sql + */ +void executeSQL(TAOS *taos, const char *sql) { + TAOS_RES *res = taos_query(taos, sql); + int code = taos_errno(res); + if (code != 0) { + printf("%s\n", taos_errstr(res)); + taos_free_result(res); + taos_close(taos); + exit(EXIT_FAILURE); + } + taos_free_result(res); +} + +/** + * @brief exit program when error occur. + * + * @param stmt + * @param code + * @param msg + */ +void checkErrorCode(TAOS_STMT *stmt, int code, const char *msg) { + if (code != 0) { + printf("%s. error: %s\n", msg, taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + exit(EXIT_FAILURE); + } +} + +/** + * @brief insert data using stmt API + * + * @param taos + */ +void insertData(TAOS *taos) { + // init + TAOS_STMT *stmt = taos_stmt_init(taos); + // prepare + const char *sql = "INSERT INTO ? USING meters TAGS(?, ?) values(?, ?, ?, ?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + checkErrorCode(stmt, code, "failed to execute taos_stmt_prepare"); + // bind table name and tags + TAOS_BIND tags[2]; + char *location = "Beijing.Chaoyang"; + int groupId = 2; + tags[0].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[0].buffer_length = strlen(location); + tags[0].length = &tags[0].buffer_length; + tags[0].buffer = location; + tags[0].is_null = NULL; + + tags[1].buffer_type = TSDB_DATA_TYPE_INT; + tags[1].buffer_length = sizeof(int); + tags[1].length = &tags[1].buffer_length; + tags[1].buffer = &groupId; + tags[1].is_null = NULL; + + code = taos_stmt_set_tbname_tags(stmt, "d1001", tags); + checkErrorCode(stmt, code, "failed to execute taos_stmt_set_tbname_tags"); + + // highlight-start + // insert two rows with multi binds + TAOS_MULTI_BIND params[4]; + // values to bind + int64_t ts[] = {1648432611249, 1648432611749}; + float current[] = {10.3, 12.6}; + int voltage[] = {219, 218}; + float phase[] = {0.31, 0.33}; + // is_null array + char is_null[2] = {0}; + // length array + int32_t int64Len[2] = {sizeof(int64_t)}; + int32_t floatLen[2] = {sizeof(float)}; + int32_t intLen[2] = {sizeof(int)}; + + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(int64_t); + params[0].buffer = ts; + params[0].length = int64Len; + params[0].is_null = is_null; + params[0].num = 2; + + params[1].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[1].buffer_length = sizeof(float); + params[1].buffer = current; + params[1].length = floatLen; + params[1].is_null = is_null; + params[1].num = 2; + + params[2].buffer_type = TSDB_DATA_TYPE_INT; + params[2].buffer_length = sizeof(int); + params[2].buffer = voltage; + params[2].length = intLen; + params[2].is_null = is_null; + params[2].num = 2; + + params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[3].buffer_length = sizeof(float); + params[3].buffer = phase; + params[3].length = floatLen; + params[3].is_null = is_null; + params[3].num = 2; + + code = taos_stmt_bind_param_batch(stmt, params); // bind batch + checkErrorCode(stmt, code, "failed to execute taos_stmt_bind_param_batch"); + code = taos_stmt_add_batch(stmt); // add batch + checkErrorCode(stmt, code, "failed to execute taos_stmt_add_batch"); + // highlight-end + // execute + code = taos_stmt_execute(stmt); + checkErrorCode(stmt, code, "failed to execute taos_stmt_execute"); + int affectedRows = taos_stmt_affected_rows(stmt); + printf("successfully inserted %d rows\n", affectedRows); + // close + taos_stmt_close(stmt); +} + +int main() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 6030); + if (taos == NULL) { + printf("failed to connect to server\n"); + exit(EXIT_FAILURE); + } + executeSQL(taos, "DROP DATABASE IF EXISTS power"); + executeSQL(taos, "CREATE DATABASE power"); + executeSQL(taos, "USE power"); + executeSQL(taos, + "CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), " + "groupId INT)"); + insertData(taos); + taos_close(taos); + taos_cleanup(); +} + +// output: +// successfully inserted 2 rows \ No newline at end of file diff --git a/docs-examples/c/query_example.c b/docs-examples/c/query_example.c new file mode 100644 index 0000000000000000000000000000000000000000..f88b2467ceb3d9bbeaf6b3beb6a24befd3e398c6 --- /dev/null +++ b/docs-examples/c/query_example.c @@ -0,0 +1,143 @@ +// compile with: +// gcc -o query_example query_example.c -ltaos +#include +#include +#include +#include +#include + +typedef int16_t VarDataLenT; + +#define TSDB_NCHAR_SIZE sizeof(int32_t) +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + +#define GET_FLOAT_VAL(x) (*(float *)(x)) +#define GET_DOUBLE_VAL(x) (*(double *)(x)) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] + +int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) { + int len = 0; + char split = ' '; + + for (int i = 0; i < numFields; ++i) { + if (i > 0) { + str[len++] = split; + } + + if (row[i] == NULL) { + len += sprintf(str + len, "%s", "NULL"); + continue; + } + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(str + len, "%d", *((int16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_INT: + len += sprintf(str + len, "%d", *((int32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UINT: + len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_FLOAT: { + float fv = 0; + fv = GET_FLOAT_VAL(row[i]); + len += sprintf(str + len, "%f", fv); + } break; + + case TSDB_DATA_TYPE_DOUBLE: { + double dv = 0; + dv = GET_DOUBLE_VAL(row[i]); + len += sprintf(str + len, "%lf", dv); + } break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); + memcpy(str + len, row[i], charLen); + len += charLen; + } break; + + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BOOL: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + default: + break; + } + } + + return len; +} + +/** + * @brief print column name and values of each row + * + * @param res + * @return int + */ +static int printResult(TAOS_RES *res) { + int numFields = taos_num_fields(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + char header[256] = {0}; + int len = 0; + for (int i = 0; i < numFields; ++i) { + len += sprintf(header + len, "%s ", fields[i].name); + } + puts(header); + + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(res))) { + char temp[256] = {0}; + printRow(temp, row, fields, numFields); + puts(temp); + } +} + +int main() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", "power", 6030); + if (taos == NULL) { + puts("failed to connect to server"); + exit(EXIT_FAILURE); + } + TAOS_RES *res = taos_query(taos, "SELECT * FROM meters LIMIT 2"); + if (taos_errno(res) != 0) { + printf("failed to execute taos_query. error: %s\n", taos_errstr(res)); + exit(EXIT_FAILURE); + } + printResult(res); + taos_free_result(res); + taos_close(taos); + taos_cleanup(); +} + +// output: +// ts current voltage phase location groupid +// 1648432611249 10.300000 219 0.310000 Beijing.Chaoyang 2 +// 1648432611749 12.600000 218 0.330000 Beijing.Chaoyang 2 \ No newline at end of file diff --git a/docs-examples/c/stmt_example.c b/docs-examples/c/stmt_example.c new file mode 100644 index 0000000000000000000000000000000000000000..fab1506f953ef68050e4318406fa2ba1a0202929 --- /dev/null +++ b/docs-examples/c/stmt_example.c @@ -0,0 +1,141 @@ +// compile with +// gcc -o stmt_example stmt_example.c -ltaos +#include +#include +#include +#include "taos.h" + +/** + * @brief execute sql only. + * + * @param taos + * @param sql + */ +void executeSQL(TAOS *taos, const char *sql) { + TAOS_RES *res = taos_query(taos, sql); + int code = taos_errno(res); + if (code != 0) { + printf("%s\n", taos_errstr(res)); + taos_free_result(res); + taos_close(taos); + exit(EXIT_FAILURE); + } + taos_free_result(res); +} + +/** + * @brief check return status and exit program when error occur. + * + * @param stmt + * @param code + * @param msg + */ +void checkErrorCode(TAOS_STMT *stmt, int code, const char* msg) { + if (code != 0) { + printf("%s. error: %s\n", msg, taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + exit(EXIT_FAILURE); + } +} + +typedef struct { + int64_t ts; + float current; + int voltage; + float phase; +} Row; + +/** + * @brief insert data using stmt API + * + * @param taos + */ +void insertData(TAOS *taos) { + // init + TAOS_STMT *stmt = taos_stmt_init(taos); + // prepare + const char *sql = "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + checkErrorCode(stmt, code, "failed to execute taos_stmt_prepare"); + // bind table name and tags + TAOS_BIND tags[2]; + char* location = "Beijing.Chaoyang"; + int groupId = 2; + tags[0].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[0].buffer_length = strlen(location); + tags[0].length = &tags[0].buffer_length; + tags[0].buffer = location; + tags[0].is_null = NULL; + + tags[1].buffer_type = TSDB_DATA_TYPE_INT; + tags[1].buffer_length = sizeof(int); + tags[1].length = &tags[1].buffer_length; + tags[1].buffer = &groupId; + tags[1].is_null = NULL; + + code = taos_stmt_set_tbname_tags(stmt, "d1001", tags); + checkErrorCode(stmt, code, "failed to execute taos_stmt_set_tbname_tags"); + + // insert two rows + Row rows[2] = { + {1648432611249, 10.3, 219, 0.31}, + {1648432611749, 12.6, 218, 0.33}, + }; + + TAOS_BIND values[4]; + values[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + values[0].buffer_length = sizeof(int64_t); + values[0].length = &values[0].buffer_length; + values[0].is_null = NULL; + + values[1].buffer_type = TSDB_DATA_TYPE_FLOAT; + values[1].buffer_length = sizeof(float); + values[1].length = &values[1].buffer_length; + values[1].is_null = NULL; + + values[2].buffer_type = TSDB_DATA_TYPE_INT; + values[2].buffer_length = sizeof(int); + values[2].length = &values[2].buffer_length; + values[2].is_null = NULL; + + values[3].buffer_type = TSDB_DATA_TYPE_FLOAT; + values[3].buffer_length = sizeof(float); + values[3].length = &values[3].buffer_length; + values[3].is_null = NULL; + + for (int i = 0; i < 2; ++i) { + values[0].buffer = &rows[i].ts; + values[1].buffer = &rows[i].current; + values[2].buffer = &rows[i].voltage; + values[3].buffer = &rows[i].phase; + code = taos_stmt_bind_param(stmt, values); // bind param + checkErrorCode(stmt, code, "failed to execute taos_stmt_bind_param"); + code = taos_stmt_add_batch(stmt); // add batch + checkErrorCode(stmt, code, "failed to execute taos_stmt_add_batch"); + } + // execute + code = taos_stmt_execute(stmt); + checkErrorCode(stmt, code, "failed to execute taos_stmt_execute"); + int affectedRows = taos_stmt_affected_rows(stmt); + printf("successfully inserted %d rows\n", affectedRows); + // close + taos_stmt_close(stmt); +} + +int main() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 6030); + if (taos == NULL) { + printf("failed to connect to server\n"); + exit(EXIT_FAILURE); + } + executeSQL(taos, "CREATE DATABASE power"); + executeSQL(taos, "USE power"); + executeSQL(taos, "CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)"); + insertData(taos); + taos_close(taos); + taos_cleanup(); +} + + +// output: +// successfully inserted 2 rows \ No newline at end of file diff --git a/docs-examples/c/subscribe_demo.c b/docs-examples/c/subscribe_demo.c new file mode 100644 index 0000000000000000000000000000000000000000..2fe62c24eb92d2f57c24b40fc16f47d62ea5e378 --- /dev/null +++ b/docs-examples/c/subscribe_demo.c @@ -0,0 +1,66 @@ +// A simple demo for asynchronous subscription. +// compile with: +// gcc -o subscribe_demo subscribe_demo.c -ltaos + +#include +#include +#include +#include + +int nTotalRows; + +/** + * @brief callback function of subscription. + * + * @param tsub + * @param res + * @param param. the additional parameter passed to taos_subscribe + * @param code. error code + */ +void subscribe_callback(TAOS_SUB* tsub, TAOS_RES* res, void* param, int code) { + if (code != 0) { + printf("error: %d\n", code); + exit(EXIT_FAILURE); + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(res); + TAOS_FIELD* fields = taos_fetch_fields(res); + int nRows = 0; + + while ((row = taos_fetch_row(res))) { + char buf[4096] = {0}; + taos_print_row(buf, row, fields, num_fields); + puts(buf); + nRows++; + } + + nTotalRows += nRows; + printf("%d rows consumed.\n", nRows); +} + +int main() { + TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 6030); + if (taos == NULL) { + printf("failed to connect to server\n"); + exit(EXIT_FAILURE); + } + + int restart = 1; // if the topic already exists, where to subscribe from the begin. + const char* topic = "topic-meter-current-bg-10"; + const char* sql = "select * from power.meters where current > 10"; + void* param = NULL; // additional parameter. + int interval = 2000; // consumption interval in microseconds. + TAOS_SUB* tsub = taos_subscribe(taos, restart, topic, sql, subscribe_callback, NULL, interval); + + // wait for insert from others process. you can open TDengine CLI to insert some records for test. + + getchar(); // press Enter to stop + + printf("total rows consumed: %d\n", nTotalRows); + int keep = 0; // whether to keep subscribe process + taos_unsubscribe(tsub, keep); + + taos_close(taos); + taos_cleanup(); +} diff --git a/docs-examples/c/telnet_line_example.c b/docs-examples/c/telnet_line_example.c new file mode 100644 index 0000000000000000000000000000000000000000..913d433f6aec07b3bce115d45536ffa4b45a0481 --- /dev/null +++ b/docs-examples/c/telnet_line_example.c @@ -0,0 +1,54 @@ +// compile with +// gcc -o telnet_line_example telnet_line_example.c -ltaos +#include +#include +#include +#include "taos.h" + +void executeSQL(TAOS *taos, const char *sql) { + TAOS_RES *res = taos_query(taos, sql); + int code = taos_errno(res); + if (code != 0) { + printf("%s\n", taos_errstr(res)); + taos_free_result(res); + taos_close(taos); + exit(EXIT_FAILURE); + } + taos_free_result(res); +} + +// ANCHOR: main +int main() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", "", 6030); + if (taos == NULL) { + printf("failed to connect to server\n"); + exit(EXIT_FAILURE); + } + executeSQL(taos, "DROP DATABASE IF EXISTS test"); + executeSQL(taos, "CREATE DATABASE test"); + executeSQL(taos, "USE test"); + char *lines[] = { + "meters.current 1648432611249 10.3 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611250 12.6 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611249 10.8 location=Beijing.Haidian groupid=3", + "meters.current 1648432611250 11.3 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611249 219 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611250 218 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611249 221 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611250 217 location=Beijing.Haidian groupid=3", + }; + TAOS_RES *res = taos_schemaless_insert(taos, lines, 8, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + if (taos_errno(res) != 0) { + printf("failed to insert schema-less data, reason: %s\n", taos_errstr(res)); + } else { + int affectedRow = taos_affected_rows(res); + printf("successfully inserted %d rows\n", affectedRow); + } + + taos_free_result(res); + taos_close(taos); + taos_cleanup(); +} +// output: +// successfully inserted 8 rows +// ANCHOR_END: main diff --git a/docs-examples/csharp/.gitignore b/docs-examples/csharp/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b3aff79f3706e23aa74199a7f521f7912d2b0e45 --- /dev/null +++ b/docs-examples/csharp/.gitignore @@ -0,0 +1,4 @@ +bin +obj +.vs +*.sln \ No newline at end of file diff --git a/docs-examples/csharp/AsyncQueryExample.cs b/docs-examples/csharp/AsyncQueryExample.cs new file mode 100644 index 0000000000000000000000000000000000000000..fe30d21efe82e8d1dc414bd4723227ca93bc944f --- /dev/null +++ b/docs-examples/csharp/AsyncQueryExample.cs @@ -0,0 +1,238 @@ +using TDengineDriver; +using System.Runtime.InteropServices; + +namespace TDengineExample +{ + public class AsyncQueryExample + { + static void Main() + { + IntPtr conn = GetConnection(); + QueryAsyncCallback queryAsyncCallback = new QueryAsyncCallback(QueryCallback); + TDengine.QueryAsync(conn, "select * from meters", queryAsyncCallback, IntPtr.Zero); + Thread.Sleep(2000); + TDengine.Close(conn); + TDengine.Cleanup(); + } + + static void QueryCallback(IntPtr param, IntPtr taosRes, int code) + { + if (code == 0 && taosRes != IntPtr.Zero) + { + FetchRowAsyncCallback fetchRowAsyncCallback = new FetchRowAsyncCallback(FetchRowCallback); + TDengine.FetchRowAsync(taosRes, fetchRowAsyncCallback, param); + } + else + { + Console.WriteLine($"async query data failed, failed code {code}"); + } + } + + static void FetchRowCallback(IntPtr param, IntPtr taosRes, int numOfRows) + { + if (numOfRows > 0) + { + Console.WriteLine($"{numOfRows} rows async retrieved"); + DisplayRes(taosRes); + TDengine.FetchRowAsync(taosRes, FetchRowCallback, param); + } + else + { + if (numOfRows == 0) + { + Console.WriteLine("async retrieve complete."); + + } + else + { + Console.WriteLine($"FetchRowAsync callback error, error code {numOfRows}"); + } + TDengine.FreeResult(taosRes); + } + } + + public static void DisplayRes(IntPtr res) + { + if (!IsValidResult(res)) + { + TDengine.Cleanup(); + System.Environment.Exit(1); + } + + List metaList = TDengine.FetchFields(res); + int fieldCount = metaList.Count; + // metaList.ForEach((item) => { Console.Write("{0} ({1}) \t|\t", item.name, item.size); }); + + List dataList = QueryRes(res, metaList); + for (int index = 0; index < dataList.Count; index++) + { + if (index % fieldCount == 0 && index != 0) + { + Console.WriteLine(""); + } + Console.Write("{0} \t|\t", dataList[index].ToString()); + + } + Console.WriteLine(""); + } + + public static bool IsValidResult(IntPtr res) + { + if ((res == IntPtr.Zero) || (TDengine.ErrorNo(res) != 0)) + { + if (res != IntPtr.Zero) + { + Console.Write("reason: " + TDengine.Error(res)); + return false; + } + Console.WriteLine(""); + return false; + } + return true; + } + + private static List QueryRes(IntPtr res, List meta) + { + IntPtr taosRow; + List dataRaw = new(); + while ((taosRow = TDengine.FetchRows(res)) != IntPtr.Zero) + { + dataRaw.AddRange(FetchRow(taosRow, res)); + } + if (TDengine.ErrorNo(res) != 0) + { + Console.Write("Query is not complete, Error {0} {1}", TDengine.ErrorNo(res), TDengine.Error(res)); + } + TDengine.FreeResult(res); + Console.WriteLine(""); + return dataRaw; + } + + public static List FetchRow(IntPtr taosRow, IntPtr taosRes)//, List metaList, int numOfFiled + { + List metaList = TDengine.FetchFields(taosRes); + int numOfFiled = TDengine.FieldCount(taosRes); + + + List dataRaw = new(); + + IntPtr colLengthPrt = TDengine.FetchLengths(taosRes); + int[] colLengthArr = new int[numOfFiled]; + Marshal.Copy(colLengthPrt, colLengthArr, 0, numOfFiled); + + for (int i = 0; i < numOfFiled; i++) + { + TDengineMeta meta = metaList[i]; + IntPtr data = Marshal.ReadIntPtr(taosRow, IntPtr.Size * i); + + if (data == IntPtr.Zero) + { + dataRaw.Add("NULL"); + continue; + } + switch ((TDengineDataType)meta.type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + bool v1 = Marshal.ReadByte(data) != 0; + dataRaw.Add(v1); + break; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + sbyte v2 = (sbyte)Marshal.ReadByte(data); + dataRaw.Add(v2); + break; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + short v3 = Marshal.ReadInt16(data); + dataRaw.Add(v3); + break; + case TDengineDataType.TSDB_DATA_TYPE_INT: + int v4 = Marshal.ReadInt32(data); + dataRaw.Add(v4); + break; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + long v5 = Marshal.ReadInt64(data); + dataRaw.Add(v5); + break; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + float v6 = (float)Marshal.PtrToStructure(data, typeof(float)); + dataRaw.Add(v6); + break; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + double v7 = (double)Marshal.PtrToStructure(data, typeof(double)); + dataRaw.Add(v7); + break; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + string v8 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); + dataRaw.Add(v8); + break; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + long v9 = Marshal.ReadInt64(data); + dataRaw.Add(v9); + break; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + string v10 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); + dataRaw.Add(v10); + break; + case TDengineDataType.TSDB_DATA_TYPE_UTINYINT: + byte v12 = Marshal.ReadByte(data); + dataRaw.Add(v12.ToString()); + break; + case TDengineDataType.TSDB_DATA_TYPE_USMALLINT: + ushort v13 = (ushort)Marshal.ReadInt16(data); + dataRaw.Add(v13); + break; + case TDengineDataType.TSDB_DATA_TYPE_UINT: + uint v14 = (uint)Marshal.ReadInt32(data); + dataRaw.Add(v14); + break; + case TDengineDataType.TSDB_DATA_TYPE_UBIGINT: + ulong v15 = (ulong)Marshal.ReadInt64(data); + dataRaw.Add(v15); + break; + case TDengineDataType.TSDB_DATA_TYPE_JSONTAG: + string v16 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); + dataRaw.Add(v16); + break; + default: + dataRaw.Add("nonsupport data type"); + break; + } + + } + return dataRaw; + } + + static IntPtr GetConnection() + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = "power"; + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + Environment.Exit(0); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + return conn; + } + } +} + +//output: +//Connect to TDengine success +//8 rows async retrieved + +//1538548685000 | 10.3 | 219 | 0.31 | beijing.chaoyang | 2 | +//1538548695000 | 12.6 | 218 | 0.33 | beijing.chaoyang | 2 | +//1538548696800 | 12.3 | 221 | 0.31 | beijing.chaoyang | 2 | +//1538548696650 | 10.3 | 218 | 0.25 | beijing.chaoyang | 3 | +//1538548685500 | 11.8 | 221 | 0.28 | beijing.haidian | 2 | +//1538548696600 | 13.4 | 223 | 0.29 | beijing.haidian | 2 | +//1538548685000 | 10.8 | 223 | 0.29 | beijing.haidian | 3 | +//1538548686500 | 11.5 | 221 | 0.35 | beijing.haidian | 3 | +//async retrieve complete. \ No newline at end of file diff --git a/docs-examples/csharp/ConnectExample.cs b/docs-examples/csharp/ConnectExample.cs new file mode 100644 index 0000000000000000000000000000000000000000..f3548ee65daab8a59695499339a8f89b0aa33a10 --- /dev/null +++ b/docs-examples/csharp/ConnectExample.cs @@ -0,0 +1,29 @@ +using TDengineDriver; + +namespace TDengineExample +{ + + internal class ConnectExample + { + static void Main(String[] args) + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = ""; + + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + TDengine.Close(conn); + TDengine.Cleanup(); + } + } +} diff --git a/docs-examples/csharp/InfluxDBLineExample.cs b/docs-examples/csharp/InfluxDBLineExample.cs new file mode 100644 index 0000000000000000000000000000000000000000..7aad08825209db568d61e5963ec7a00034ab7ca7 --- /dev/null +++ b/docs-examples/csharp/InfluxDBLineExample.cs @@ -0,0 +1,77 @@ +using TDengineDriver; + +namespace TDengineExample +{ + internal class InfluxDBLineExample + { + static void Main() + { + IntPtr conn = GetConnection(); + PrepareDatabase(conn); + string[] lines = { + "meters,location=Beijing.Haidian,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249", + "meters,location=Beijing.Haidian,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250", + "meters,location=Beijing.Haidian,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249", + "meters,location=Beijing.Haidian,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250" + }; + IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS); + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine("SchemalessInsert failed since " + TDengine.Error(res)); + ExitProgram(conn, 1); + } + else + { + int affectedRows = TDengine.AffectRows(res); + Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); + } + TDengine.FreeResult(res); + ExitProgram(conn, 0); + + } + static IntPtr GetConnection() + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = ""; + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + TDengine.Cleanup(); + Environment.Exit(1); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + return conn; + } + + static void PrepareDatabase(IntPtr conn) + { + IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine("failed to create database, reason: " + TDengine.Error(res)); + ExitProgram(conn, 1); + } + res = TDengine.Query(conn, "USE test"); + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine("failed to change database, reason: " + TDengine.Error(res)); + ExitProgram(conn, 1); + } + } + + static void ExitProgram(IntPtr conn, int exitCode) + { + TDengine.Close(conn); + TDengine.Cleanup(); + Environment.Exit(exitCode); + } + } + +} diff --git a/docs-examples/csharp/OptsJsonExample.cs b/docs-examples/csharp/OptsJsonExample.cs new file mode 100644 index 0000000000000000000000000000000000000000..d774a325afa1a8d93eb858f23dcd97dd29f8653d --- /dev/null +++ b/docs-examples/csharp/OptsJsonExample.cs @@ -0,0 +1,76 @@ +using TDengineDriver; + +namespace TDengineExample +{ + internal class OptsJsonExample + { + static void Main() + { + IntPtr conn = GetConnection(); + PrepareDatabase(conn); + string[] lines = { "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"Beijing.Chaoyang\", \"groupid\": 2}}," + + " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"Beijing.Haidian\", \"groupid\": 1}}, " + + "{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"Beijing.Chaoyang\", \"groupid\": 2}}," + + " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"Beijing.Haidian\", \"groupid\": 1}}]" + }; + + IntPtr res = TDengine.SchemalessInsert(conn, lines, 1, (int)TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine("SchemalessInsert failed since " + TDengine.Error(res)); + ExitProgram(conn, 1); + } + else + { + int affectedRows = TDengine.AffectRows(res); + Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); + } + TDengine.FreeResult(res); + ExitProgram(conn, 0); + + } + static IntPtr GetConnection() + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = ""; + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + TDengine.Cleanup(); + Environment.Exit(1); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + return conn; + } + + static void PrepareDatabase(IntPtr conn) + { + IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine("failed to create database, reason: " + TDengine.Error(res)); + ExitProgram(conn, 1); + } + res = TDengine.Query(conn, "USE test"); + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine("failed to change database, reason: " + TDengine.Error(res)); + ExitProgram(conn, 1); + } + } + + static void ExitProgram(IntPtr conn, int exitCode) + { + TDengine.Close(conn); + TDengine.Cleanup(); + Environment.Exit(exitCode); + } + } +} diff --git a/docs-examples/csharp/OptsTelnetExample.cs b/docs-examples/csharp/OptsTelnetExample.cs new file mode 100644 index 0000000000000000000000000000000000000000..81608c32213fa0618a2ca6e0769aacf8e9c8e64d --- /dev/null +++ b/docs-examples/csharp/OptsTelnetExample.cs @@ -0,0 +1,80 @@ +using TDengineDriver; + +namespace TDengineExample +{ + internal class OptsTelnetExample + { + static void Main() + { + IntPtr conn = GetConnection(); + PrepareDatabase(conn); + string[] lines = { + "meters.current 1648432611249 10.3 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611250 12.6 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611249 10.8 location=Beijing.Haidian groupid=3", + "meters.current 1648432611250 11.3 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611249 219 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611250 218 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611249 221 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611250 217 location=Beijing.Haidian groupid=3", + }; + IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine("SchemalessInsert failed since " + TDengine.Error(res)); + ExitProgram(conn, 1); + } + else + { + int affectedRows = TDengine.AffectRows(res); + Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows"); + } + TDengine.FreeResult(res); + ExitProgram(conn, 0); + + } + static IntPtr GetConnection() + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = ""; + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + TDengine.Cleanup(); + Environment.Exit(1); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + return conn; + } + + static void PrepareDatabase(IntPtr conn) + { + IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine("failed to create database, reason: " + TDengine.Error(res)); + ExitProgram(conn, 1); + } + res = TDengine.Query(conn, "USE test"); + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine("failed to change database, reason: " + TDengine.Error(res)); + ExitProgram(conn, 1); + } + } + + static void ExitProgram(IntPtr conn, int exitCode) + { + TDengine.Close(conn); + TDengine.Cleanup(); + Environment.Exit(exitCode); + } + } +} diff --git a/docs-examples/csharp/QueryExample.cs b/docs-examples/csharp/QueryExample.cs new file mode 100644 index 0000000000000000000000000000000000000000..f00e391100c7ce42177e2987f5b0b32dc02262c4 --- /dev/null +++ b/docs-examples/csharp/QueryExample.cs @@ -0,0 +1,162 @@ +using TDengineDriver; +using System.Runtime.InteropServices; + +namespace TDengineExample +{ + internal class QueryExample + { + static void Main() + { + IntPtr conn = GetConnection(); + // run query + IntPtr res = TDengine.Query(conn, "SELECT * FROM test.meters LIMIT 2"); + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine("Failed to query since: " + TDengine.Error(res)); + TDengine.Close(conn); + TDengine.Cleanup(); + return; + } + + // get filed count + int fieldCount = TDengine.FieldCount(res); + Console.WriteLine("fieldCount=" + fieldCount); + + // print column names + List metas = TDengine.FetchFields(res); + for (int i = 0; i < metas.Count; i++) + { + Console.Write(metas[i].name + "\t"); + } + Console.WriteLine(); + + // print values + IntPtr row; + while ((row = TDengine.FetchRows(res)) != IntPtr.Zero) + { + List metaList = TDengine.FetchFields(res); + int numOfFiled = TDengine.FieldCount(res); + + List dataRaw = new List(); + + IntPtr colLengthPrt = TDengine.FetchLengths(res); + int[] colLengthArr = new int[numOfFiled]; + Marshal.Copy(colLengthPrt, colLengthArr, 0, numOfFiled); + + for (int i = 0; i < numOfFiled; i++) + { + TDengineMeta meta = metaList[i]; + IntPtr data = Marshal.ReadIntPtr(row, IntPtr.Size * i); + + if (data == IntPtr.Zero) + { + Console.Write("NULL\t"); + continue; + } + switch ((TDengineDataType)meta.type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + bool v1 = Marshal.ReadByte(data) == 0 ? false : true; + Console.Write(v1.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + sbyte v2 = (sbyte)Marshal.ReadByte(data); + Console.Write(v2.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + short v3 = Marshal.ReadInt16(data); + Console.Write(v3.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_INT: + int v4 = Marshal.ReadInt32(data); + Console.Write(v4.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + long v5 = Marshal.ReadInt64(data); + Console.Write(v5.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + float v6 = (float)Marshal.PtrToStructure(data, typeof(float)); + Console.Write(v6.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + double v7 = (double)Marshal.PtrToStructure(data, typeof(double)); + Console.Write(v7.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + string v8 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); + Console.Write(v8 + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + long v9 = Marshal.ReadInt64(data); + Console.Write(v9.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + string v10 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); + Console.Write(v10 + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_UTINYINT: + byte v12 = Marshal.ReadByte(data); + Console.Write(v12.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_USMALLINT: + ushort v13 = (ushort)Marshal.ReadInt16(data); + Console.Write(v13.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_UINT: + uint v14 = (uint)Marshal.ReadInt32(data); + Console.Write(v14.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_UBIGINT: + ulong v15 = (ulong)Marshal.ReadInt64(data); + Console.Write(v15.ToString() + "\t"); + break; + case TDengineDataType.TSDB_DATA_TYPE_JSONTAG: + string v16 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); + Console.Write(v16 + "\t"); + break; + default: + Console.Write("nonsupport data type value"); + break; + } + + } + Console.WriteLine(); + } + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine($"Query is not complete, Error {TDengine.ErrorNo(res)} {TDengine.Error(res)}"); + } + // exit + TDengine.FreeResult(res); + TDengine.Close(conn); + TDengine.Cleanup(); + } + static IntPtr GetConnection() + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = "power"; + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + System.Environment.Exit(0); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + return conn; + } + } +} + +// output: +// Connect to TDengine success +// fieldCount=6 +// ts current voltage phase location groupid +// 1648432611249 10.3 219 0.31 Beijing.Chaoyang 2 +// 1648432611749 12.6 218 0.33 Beijing.Chaoyang 2 \ No newline at end of file diff --git a/docs-examples/csharp/SQLInsertExample.cs b/docs-examples/csharp/SQLInsertExample.cs new file mode 100644 index 0000000000000000000000000000000000000000..fa2e2a50daf06f4d948479e7f5b0df82c517f809 --- /dev/null +++ b/docs-examples/csharp/SQLInsertExample.cs @@ -0,0 +1,69 @@ +using TDengineDriver; + + +namespace TDengineExample +{ + internal class SQLInsertExample + { + + static void Main() + { + IntPtr conn = GetConnection(); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE power"); + CheckRes(conn, res, "failed to create database"); + res = TDengine.Query(conn, "USE power"); + CheckRes(conn, res, "failed to change database"); + res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)"); + CheckRes(conn, res, "failed to create stable"); + var sql = "INSERT INTO d1001 USING meters TAGS(Beijing.Chaoyang, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) " + + "d1002 USING power.meters TAGS(Beijing.Chaoyang, 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) " + + "d1003 USING power.meters TAGS(Beijing.Haidian, 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " + + "d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; + res = TDengine.Query(conn, sql); + CheckRes(conn, res, "failed to insert data"); + int affectedRows = TDengine.AffectRows(res); + Console.WriteLine("affectedRows " + affectedRows); + ExitProgram(conn, 0); + } + + static IntPtr GetConnection() + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = ""; + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + Environment.Exit(0); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + return conn; + } + + static void CheckRes(IntPtr conn, IntPtr res, String errorMsg) + { + if (TDengine.ErrorNo(res) != 0) + { + Console.Write(errorMsg + " since: " + TDengine.Error(res)); + ExitProgram(conn, 1); + } + } + + static void ExitProgram(IntPtr conn, int exitCode) + { + TDengine.Close(conn); + TDengine.Cleanup(); + Environment.Exit(exitCode); + } + } +} + +// output: +// Connect to TDengine success +// affectedRows 8 diff --git a/docs-examples/csharp/StmtInsertExample.cs b/docs-examples/csharp/StmtInsertExample.cs new file mode 100644 index 0000000000000000000000000000000000000000..d6e00dd4ac54ab8dbfc33b93896d19fc585e7642 --- /dev/null +++ b/docs-examples/csharp/StmtInsertExample.cs @@ -0,0 +1,115 @@ +using TDengineDriver; + +namespace TDengineExample +{ + internal class StmtInsertExample + { + private static IntPtr conn; + private static IntPtr stmt; + static void Main() + { + conn = GetConnection(); + PrepareSTable(); + // 1. init and prepare + stmt = TDengine.StmtInit(conn); + if (stmt == IntPtr.Zero) + { + Console.WriteLine("failed to init stmt, " + TDengine.Error(stmt)); + ExitProgram(); + } + int res = TDengine.StmtPrepare(stmt, "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)"); + CheckStmtRes(res, "failed to prepare stmt"); + + // 2. bind table name and tags + TAOS_BIND[] tags = new TAOS_BIND[2] { TaosBind.BindBinary("Beijing.Chaoyang"), TaosBind.BindInt(2) }; + res = TDengine.StmtSetTbnameTags(stmt, "d1001", tags); + CheckStmtRes(res, "failed to bind table name and tags"); + + // 3. bind values + TAOS_MULTI_BIND[] values = new TAOS_MULTI_BIND[4] { + TaosMultiBind.MultiBindTimestamp(new long[2] { 1648432611249, 1648432611749}), + TaosMultiBind.MultiBindFloat(new float?[2] { 10.3f, 12.6f}), + TaosMultiBind.MultiBindInt(new int?[2] { 219, 218}), + TaosMultiBind.MultiBindFloat(new float?[2]{ 0.31f, 0.33f}) + }; + res = TDengine.StmtBindParamBatch(stmt, values); + CheckStmtRes(res, "failed to bind params"); + + // 4. add batch + res = TDengine.StmtAddBatch(stmt); + CheckStmtRes(res, "failed to add batch"); + + // 5. execute + res = TDengine.StmtExecute(stmt); + CheckStmtRes(res, "faild to execute"); + + // 6. free + TaosBind.FreeTaosBind(tags); + TaosMultiBind.FreeTaosBind(values); + TDengine.Close(conn); + TDengine.Cleanup(); + } + + static IntPtr GetConnection() + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = ""; + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + Environment.Exit(0); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + return conn; + } + + + + static void PrepareSTable() + { + IntPtr res = TDengine.Query(conn, "CREATE DATABASE power"); + CheckResPtr(res, "failed to create database"); + res = TDengine.Query(conn, "USE power"); + CheckResPtr(res, "failed to change database"); + res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)"); + CheckResPtr(res, "failed to create stable"); + } + + static void CheckStmtRes(int res, string errorMsg) + { + if (res != 0) + { + Console.WriteLine(errorMsg + ", " + TDengine.StmtErrorStr(stmt)); + int code = TDengine.StmtClose(stmt); + if (code != 0) + { + Console.WriteLine($"falied to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); + } + ExitProgram(); + } + } + + static void CheckResPtr(IntPtr res, string errorMsg) + { + if (TDengine.ErrorNo(res) != 0) + { + Console.WriteLine(errorMsg + " since:" + TDengine.Error(res)); + ExitProgram(); + } + } + + static void ExitProgram() + { + TDengine.Close(conn); + TDengine.Cleanup(); + Environment.Exit(1); + } + } +} diff --git a/docs-examples/csharp/SubscribeDemo.cs b/docs-examples/csharp/SubscribeDemo.cs new file mode 100644 index 0000000000000000000000000000000000000000..34509215da73ea6369eb95f458d622cd95a97932 --- /dev/null +++ b/docs-examples/csharp/SubscribeDemo.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace csharp +{ + internal class SubscribeDemo + { + } +} diff --git a/docs-examples/csharp/asyncquery.csproj b/docs-examples/csharp/asyncquery.csproj new file mode 100644 index 0000000000000000000000000000000000000000..7a952fe7abd15798bc704de52aa72624e59061eb --- /dev/null +++ b/docs-examples/csharp/asyncquery.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + enable + enable + TDengineExample.AsyncQueryExample + + + + + + + diff --git a/docs-examples/csharp/connect.csproj b/docs-examples/csharp/connect.csproj new file mode 100644 index 0000000000000000000000000000000000000000..27cffa30ae5a98277b74c63cf3d2c749ec67ce8d --- /dev/null +++ b/docs-examples/csharp/connect.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + enable + enable + TDengineExample.ConnectExample + + + + + + + diff --git a/docs-examples/csharp/influxdbline.csproj b/docs-examples/csharp/influxdbline.csproj new file mode 100644 index 0000000000000000000000000000000000000000..a8b197dc7167f229b903bfd010a9ff7282c9173b --- /dev/null +++ b/docs-examples/csharp/influxdbline.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + enable + enable + TDengineExample.InfluxDBLineExample + + + + + + + diff --git a/docs-examples/csharp/optsjson.csproj b/docs-examples/csharp/optsjson.csproj new file mode 100644 index 0000000000000000000000000000000000000000..b1bd83405efb1086b23134a9c875912a9c09e3f8 --- /dev/null +++ b/docs-examples/csharp/optsjson.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + enable + enable + TDengineExample.OptsJsonExample + + + + + + + diff --git a/docs-examples/csharp/optstelnet.csproj b/docs-examples/csharp/optstelnet.csproj new file mode 100644 index 0000000000000000000000000000000000000000..1ab41067715f8cb070e78aec2be59d686bac0e86 --- /dev/null +++ b/docs-examples/csharp/optstelnet.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + enable + enable + TDengineExample.OptsTelnetExample + + + + + + + diff --git a/docs-examples/csharp/query.csproj b/docs-examples/csharp/query.csproj new file mode 100644 index 0000000000000000000000000000000000000000..63f13c3ddbf13be9ffe5c39054cc6e4cae6001e7 --- /dev/null +++ b/docs-examples/csharp/query.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + enable + enable + TDengineExample.QueryExample + + + + + + + diff --git a/docs-examples/csharp/sqlinsert.csproj b/docs-examples/csharp/sqlinsert.csproj new file mode 100644 index 0000000000000000000000000000000000000000..0380395a5ad00d96418410b2f7974b294bf2bb07 --- /dev/null +++ b/docs-examples/csharp/sqlinsert.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + enable + enable + TDengineExample.SQLInsertExample + + + + + + + diff --git a/docs-examples/csharp/stmtinsert.csproj b/docs-examples/csharp/stmtinsert.csproj new file mode 100644 index 0000000000000000000000000000000000000000..8defb895eb2894c680fe1b9dd7346223f0f92df1 --- /dev/null +++ b/docs-examples/csharp/stmtinsert.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + enable + enable + TDengineExample.StmtInsertExample + + + + + + + diff --git a/docs-examples/csharp/subscribe.csproj b/docs-examples/csharp/subscribe.csproj new file mode 100644 index 0000000000000000000000000000000000000000..8286922c6f926b166f14983525e604ce9e13d74a --- /dev/null +++ b/docs-examples/csharp/subscribe.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + enable + enable + TDengineExample.SubscribeDemo + + + + + + + diff --git a/docs-examples/go/.gitignore b/docs-examples/go/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b6128386c306834cddeb425f365720bb9c9ce790 --- /dev/null +++ b/docs-examples/go/.gitignore @@ -0,0 +1,3 @@ +.idea +.vscode +tmp/ \ No newline at end of file diff --git a/docs-examples/go/connect/afconn/main.go b/docs-examples/go/connect/afconn/main.go new file mode 100644 index 0000000000000000000000000000000000000000..2d36ef03ab049a313ed0416f552397ff252e022e --- /dev/null +++ b/docs-examples/go/connect/afconn/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "fmt" + + "github.com/taosdata/driver-go/v2/af" +) + +func main() { + conn, err := af.Open("localhost", "root", "taosdata", "", 6030) + defer conn.Close() + if err != nil { + fmt.Println("failed to connect, err:", err) + } else { + fmt.Println("connected") + } +} diff --git a/docs-examples/go/connect/cgoexample/main.go b/docs-examples/go/connect/cgoexample/main.go new file mode 100644 index 0000000000000000000000000000000000000000..8b9aba4ce4217c00605bc8796c788f3dd52805e6 --- /dev/null +++ b/docs-examples/go/connect/cgoexample/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "database/sql" + "fmt" + + _ "github.com/taosdata/driver-go/v2/taosSql" +) + +func main() { + var taosDSN = "root:taosdata@tcp(localhost:6030)/" + taos, err := sql.Open("taosSql", taosDSN) + if err != nil { + fmt.Println("failed to connect TDengine, err:", err) + return + } + fmt.Println("Connected") + defer taos.Close() +} + +// use +// var taosDSN = "root:taosdata@tcp(localhost:6030)/dbName" +// if you want to connect to a default database. diff --git a/docs-examples/go/connect/restexample/main.go b/docs-examples/go/connect/restexample/main.go new file mode 100644 index 0000000000000000000000000000000000000000..9c05e7eed80dee4ae7e6b20637d265f388d7438d --- /dev/null +++ b/docs-examples/go/connect/restexample/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "database/sql" + "fmt" + + _ "github.com/taosdata/driver-go/v2/taosRestful" +) + +func main() { + var taosDSN = "root:taosdata@http(localhost:6041)/" + taos, err := sql.Open("taosRestful", taosDSN) + if err != nil { + fmt.Println("failed to connect TDengine, err:", err) + return + } + fmt.Println("Connected") + defer taos.Close() +} + +// use +// var taosDSN = "root:taosdata@http(localhost:6041)/dbName" +// if you want to connect to a default database. diff --git a/docs-examples/go/connect/wrapper/main.go b/docs-examples/go/connect/wrapper/main.go new file mode 100644 index 0000000000000000000000000000000000000000..d7e71a8baf229012b6944aad0eaac232924dd667 --- /dev/null +++ b/docs-examples/go/connect/wrapper/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "fmt" + + "github.com/taosdata/driver-go/v2/wrapper" +) + +func main() { + conn, err := wrapper.TaosConnect("localhost", "root", "taosdata", "", 6030) + defer wrapper.TaosClose(conn) + if err != nil { + fmt.Println("fail to connect, err:", err) + } else { + fmt.Println("connected") + } +} diff --git a/docs-examples/go/go.mod b/docs-examples/go/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..5945e395e93b373d47fe71f3584c37fed9526638 --- /dev/null +++ b/docs-examples/go/go.mod @@ -0,0 +1,6 @@ +module goexample + +go 1.17 + +require github.com/taosdata/driver-go/v2 develop + diff --git a/docs-examples/go/insert/json/main.go b/docs-examples/go/insert/json/main.go new file mode 100644 index 0000000000000000000000000000000000000000..47d9e9984adc05896fb9954ad3deffde3764b836 --- /dev/null +++ b/docs-examples/go/insert/json/main.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + + "github.com/taosdata/driver-go/v2/af" +) + +func prepareDatabase(conn *af.Connector) { + _, err := conn.Exec("CREATE DATABASE test") + if err != nil { + panic(err) + } + _, err = conn.Exec("USE test") + if err != nil { + panic(err) + } +} + +func main() { + conn, err := af.Open("localhost", "root", "taosdata", "", 6030) + if err != nil { + fmt.Println("fail to connect, err:", err) + } + defer conn.Close() + prepareDatabase(conn) + + payload := `[{"metric": "meters.current", "timestamp": 1648432611249, "value": 10.3, "tags": {"location": "Beijing.Chaoyang", "groupid": 2}}, + {"metric": "meters.voltage", "timestamp": 1648432611249, "value": 219, "tags": {"location": "Beijing.Haidian", "groupid": 1}}, + {"metric": "meters.current", "timestamp": 1648432611250, "value": 12.6, "tags": {"location": "Beijing.Chaoyang", "groupid": 2}}, + {"metric": "meters.voltage", "timestamp": 1648432611250, "value": 221, "tags": {"location": "Beijing.Haidian", "groupid": 1}}]` + + err = conn.OpenTSDBInsertJsonPayload(payload) + if err != nil { + fmt.Println("insert error:", err) + } +} diff --git a/docs-examples/go/insert/line/main.go b/docs-examples/go/insert/line/main.go new file mode 100644 index 0000000000000000000000000000000000000000..bbc41468fe5f13d3e6f896445bb88f3eba584d0f --- /dev/null +++ b/docs-examples/go/insert/line/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + + "github.com/taosdata/driver-go/v2/af" +) + +func prepareDatabase(conn *af.Connector) { + _, err := conn.Exec("CREATE DATABASE test") + if err != nil { + panic(err) + } + _, err = conn.Exec("USE test") + if err != nil { + panic(err) + } +} + +func main() { + conn, err := af.Open("localhost", "root", "taosdata", "", 6030) + if err != nil { + fmt.Println("fail to connect, err:", err) + } + defer conn.Close() + prepareDatabase(conn) + var lines = []string{ + "meters,location=Beijing.Haidian,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249", + "meters,location=Beijing.Haidian,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250", + "meters,location=Beijing.Haidian,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249", + "meters,location=Beijing.Haidian,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250", + } + + err = conn.InfluxDBInsertLines(lines, "ms") + if err != nil { + fmt.Println("insert error:", err) + } +} diff --git a/docs-examples/go/insert/sql/main.go b/docs-examples/go/insert/sql/main.go new file mode 100644 index 0000000000000000000000000000000000000000..91386855334c1930af721e0b4f43395c6a6d8e82 --- /dev/null +++ b/docs-examples/go/insert/sql/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "database/sql" + "fmt" + + _ "github.com/taosdata/driver-go/v2/taosRestful" +) + +func createStable(taos *sql.DB) { + _, err := taos.Exec("CREATE DATABASE power") + if err != nil { + fmt.Println("failed to create database, err:", err) + } + _, err = taos.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") + if err != nil { + fmt.Println("failed to create stable, err:", err) + } +} + +func insertData(taos *sql.DB) { + sql := `INSERT INTO power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS(Beijing.Chaoyang, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS(Beijing.Haidian, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)` + result, err := taos.Exec(sql) + if err != nil { + fmt.Println("failed to insert, err:", err) + return + } + rowsAffected, err := result.RowsAffected() + if err != nil { + fmt.Println("failed to get affected rows, err:", err) + return + } + fmt.Println("RowsAffected", rowsAffected) +} + +func main() { + var taosDSN = "root:taosdata@http(localhost:6041)/" + taos, err := sql.Open("taosRestful", taosDSN) + if err != nil { + fmt.Println("failed to connect TDengine, err:", err) + return + } + defer taos.Close() + createStable(taos) + insertData(taos) +} diff --git a/docs-examples/go/insert/stmt/main.go b/docs-examples/go/insert/stmt/main.go new file mode 100644 index 0000000000000000000000000000000000000000..c50200ebb427c4c64c2737cb8fe4c3d287551a34 --- /dev/null +++ b/docs-examples/go/insert/stmt/main.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "time" + + "github.com/taosdata/driver-go/v2/af" + "github.com/taosdata/driver-go/v2/af/param" + "github.com/taosdata/driver-go/v2/common" +) + +func checkErr(err error, prompt string) { + if err != nil { + fmt.Printf("%s\n", prompt) + panic(err) + } +} + +func prepareStable(conn *af.Connector) { + _, err := conn.Exec("CREATE DATABASE power") + checkErr(err, "failed to create database") + _, err = conn.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") + checkErr(err, "failed to create stable") + _, err = conn.Exec("USE power") + checkErr(err, "failed to change database") +} + +func main() { + conn, err := af.Open("localhost", "root", "taosdata", "", 6030) + checkErr(err, "fail to connect") + defer conn.Close() + prepareStable(conn) + // create stmt + stmt := conn.InsertStmt() + defer stmt.Close() + err = stmt.Prepare("INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)") + checkErr(err, "failed to create prepare statement") + + // bind table name and tags + tagParams := param.NewParam(2).AddBinary([]byte("Beijing.Chaoyang")).AddInt(2) + err = stmt.SetTableNameWithTags("d1001", tagParams) + checkErr(err, "failed to execute SetTableNameWithTags") + + // specify ColumnType + var bindType *param.ColumnType = param.NewColumnType(4).AddTimestamp().AddFloat().AddInt().AddFloat() + + // bind values. note: can only bind one row each time. + valueParams := []*param.Param{ + param.NewParam(1).AddTimestamp(time.Unix(1648432611, 249300000), common.PrecisionMilliSecond), + param.NewParam(1).AddFloat(10.3), + param.NewParam(1).AddInt(219), + param.NewParam(1).AddFloat(0.31), + } + err = stmt.BindParam(valueParams, bindType) + checkErr(err, "BindParam error") + err = stmt.AddBatch() + checkErr(err, "AddBatch error") + + // bind one more row + valueParams = []*param.Param{ + param.NewParam(1).AddTimestamp(time.Unix(1648432611, 749300000), common.PrecisionMilliSecond), + param.NewParam(1).AddFloat(12.6), + param.NewParam(1).AddInt(218), + param.NewParam(1).AddFloat(0.33), + } + err = stmt.BindParam(valueParams, bindType) + checkErr(err, "BindParam error") + err = stmt.AddBatch() + checkErr(err, "AddBatch error") + // execute + err = stmt.Execute() + checkErr(err, "Execute batch error") +} diff --git a/docs-examples/go/insert/telnet/main.go b/docs-examples/go/insert/telnet/main.go new file mode 100644 index 0000000000000000000000000000000000000000..879e6d5cece74fd0b7c815dd34614dca3c9d4544 --- /dev/null +++ b/docs-examples/go/insert/telnet/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "fmt" + + "github.com/taosdata/driver-go/v2/af" +) + +func prepareDatabase(conn *af.Connector) { + _, err := conn.Exec("CREATE DATABASE test") + if err != nil { + panic(err) + } + _, err = conn.Exec("USE test") + if err != nil { + panic(err) + } +} + +func main() { + conn, err := af.Open("localhost", "root", "taosdata", "", 6030) + if err != nil { + fmt.Println("fail to connect, err:", err) + } + defer conn.Close() + prepareDatabase(conn) + var lines = []string{ + "meters.current 1648432611249 10.3 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611250 12.6 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611249 10.8 location=Beijing.Haidian groupid=3", + "meters.current 1648432611250 11.3 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611249 219 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611250 218 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611249 221 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611250 217 location=Beijing.Haidian groupid=3", + } + + err = conn.OpenTSDBInsertTelnetLines(lines) + if err != nil { + fmt.Println("insert error:", err) + } +} diff --git a/docs-examples/go/query/async/main.go b/docs-examples/go/query/async/main.go new file mode 100644 index 0000000000000000000000000000000000000000..7776ec8d3f268a3ef393d2097b71dc5681cf48d9 --- /dev/null +++ b/docs-examples/go/query/async/main.go @@ -0,0 +1,7 @@ +package main + +import "fmt" + +func main() { + fmt.Println("hello world!") +} diff --git a/docs-examples/go/query/sync/main.go b/docs-examples/go/query/sync/main.go new file mode 100644 index 0000000000000000000000000000000000000000..0326c7a7da5da212f66b1421bb46e5972bef9609 --- /dev/null +++ b/docs-examples/go/query/sync/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "database/sql" + "fmt" + "time" + + _ "github.com/taosdata/driver-go/v2/taosRestful" +) + +func main() { + var taosDSN = "root:taosdata@http(localhost:6041)/power" + taos, err := sql.Open("taosRestful", taosDSN) + if err != nil { + fmt.Println("failed to connect TDengine, err:", err) + return + } + defer taos.Close() + rows, err := taos.Query("SELECT ts, current FROM meters LIMIT 2") + if err != nil { + fmt.Println("failed to select from table, err:", err) + return + } + + defer rows.Close() + for rows.Next() { + var r struct { + ts time.Time + current float32 + } + err := rows.Scan(&r.ts, &r.current) + if err != nil { + fmt.Println("scan error:\n", err) + return + } + fmt.Println(r.ts, r.current) + } +} diff --git a/docs-examples/go/rest/opentsdbjson/main.go b/docs-examples/go/rest/opentsdbjson/main.go new file mode 100644 index 0000000000000000000000000000000000000000..f4f5792e29436e539f301c14ce76607ad3e5593d --- /dev/null +++ b/docs-examples/go/rest/opentsdbjson/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "time" +) + +type Tags struct { + Location string `json:"location"` + Groupid int32 `json:"groupid"` +} + +type Metric struct { + Metric string `json:"metric"` + Timestamp int64 `json:"timestamp"` + Value int32 `json:"value"` + Tags Tags `json:"tags"` +} + +func main() { + client := http.Client{} + for i := 0; i < 10; i++ { + metric := Metric{"voltage", time.Now().UnixMilli(), 1, Tags{"A", 1}} + json, err := json.Marshal(metric) + if err != nil { + panic(err) + } + req, err := http.NewRequest("POST", "http://localhost:6041/opentsdb/v1/put/json/rest_go", bytes.NewBuffer(json)) + if err != nil { + panic(err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Basic cm9vdDp0YW9zZGF0YQ==") + resp, err := client.Do(req) + if err != nil { + panic(err) + } + fmt.Printf("%v\n", resp) + time.Sleep(time.Second) + + } +} diff --git a/docs-examples/go/sub/main.go b/docs-examples/go/sub/main.go new file mode 100644 index 0000000000000000000000000000000000000000..c44521b34026ebf6f84f70022a7c2d1f83e9b781 --- /dev/null +++ b/docs-examples/go/sub/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "database/sql/driver" + "fmt" + "io" + "os" + "time" + + taos "github.com/taosdata/driver-go/v2/af" +) + +func main() { + db, err := taos.Open("", "", "", "log", 0) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer db.Close() + topic, err := db.Subscribe(false, "taoslogtail", "select ts, level, ipaddr, content from log", time.Second) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + defer topic.Unsubscribe(true) + for { + func() { + rows, err := topic.Consume() + defer func() { rows.Close(); time.Sleep(time.Second) }() + if err != nil { + fmt.Println(err) + os.Exit(3) + } + for { + values := make([]driver.Value, 4) + err := rows.Next(values) + if err == io.EOF { + break + } else if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(4) + } + ts := values[0].(time.Time) + level := values[1].(int8) + ipaddr := values[2].(string) + content := values[3].(string) + fmt.Printf("%s %d %s %s\n", ts.Format(time.StampMilli), level, ipaddr, content) + } + }() + } +} + +// 未完成 diff --git a/docs-examples/java/.gitignore b/docs-examples/java/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..d48c759d6c990444f193c0043ec9b316029d31bc --- /dev/null +++ b/docs-examples/java/.gitignore @@ -0,0 +1,2 @@ +.idea +.vscode \ No newline at end of file diff --git a/docs-examples/java/pom.xml b/docs-examples/java/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..a48ba398da92f401235819d067aa2ba6f8b173ea --- /dev/null +++ b/docs-examples/java/pom.xml @@ -0,0 +1,35 @@ + + + + 4.0.0 + + com.taos + javaexample + 1.0 + + JavaExample + + + UTF-8 + 1.8 + 1.8 + + + + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.38 + + + + junit + junit + 4.13.1 + test + + + + diff --git a/docs-examples/java/src/main/java/com/taos/example/JNIConnectExample.java b/docs-examples/java/src/main/java/com/taos/example/JNIConnectExample.java new file mode 100644 index 0000000000000000000000000000000000000000..c6ce2ef9785a010daa55ad29415f81711760cd57 --- /dev/null +++ b/docs-examples/java/src/main/java/com/taos/example/JNIConnectExample.java @@ -0,0 +1,25 @@ +package com.taos.example; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +import com.taosdata.jdbc.TSDBDriver; + +public class JNIConnectExample { + public static void main(String[] args) throws SQLException { + String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + System.out.println("Connected"); + conn.close(); + } +} + +// use +// String jdbcUrl = "jdbc:TAOS://localhost:6030/dbName?user=root&password=taosdata"; +// if you want to connect to a default database. \ No newline at end of file diff --git a/docs-examples/java/src/main/java/com/taos/example/JSONProtocolExample.java b/docs-examples/java/src/main/java/com/taos/example/JSONProtocolExample.java new file mode 100644 index 0000000000000000000000000000000000000000..cb83424576a4fd7dfa09ea297294ed77b66bd12d --- /dev/null +++ b/docs-examples/java/src/main/java/com/taos/example/JSONProtocolExample.java @@ -0,0 +1,40 @@ +package com.taos.example; + +import com.taosdata.jdbc.SchemalessWriter; +import com.taosdata.jdbc.enums.SchemalessProtocolType; +import com.taosdata.jdbc.enums.SchemalessTimestampType; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +public class JSONProtocolExample { + private static Connection getConnection() throws SQLException { + String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata"; + return DriverManager.getConnection(jdbcUrl); + } + + private static void createDatabase(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE DATABASE IF NOT EXISTS test"); + stmt.execute("USE test"); + } + } + + private static String getJSONData() { + return "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"Beijing.Chaoyang\", \"groupid\": 2}}," + + " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"Beijing.Haidian\", \"groupid\": 1}}, " + + "{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"Beijing.Chaoyang\", \"groupid\": 2}}," + + " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"Beijing.Haidian\", \"groupid\": 1}}]"; + } + + public static void main(String[] args) throws SQLException { + try (Connection conn = getConnection()) { + createDatabase(conn); + SchemalessWriter writer = new SchemalessWriter(conn); + String jsonData = getJSONData(); + writer.write(jsonData, SchemalessProtocolType.JSON, SchemalessTimestampType.NOT_CONFIGURED); + } + } +} diff --git a/docs-examples/java/src/main/java/com/taos/example/LineProtocolExample.java b/docs-examples/java/src/main/java/com/taos/example/LineProtocolExample.java new file mode 100644 index 0000000000000000000000000000000000000000..8a2eabe0a91f7966cc3cc6b7dfeeb71b71b88d92 --- /dev/null +++ b/docs-examples/java/src/main/java/com/taos/example/LineProtocolExample.java @@ -0,0 +1,42 @@ +package com.taos.example; + +import com.taosdata.jdbc.SchemalessWriter; +import com.taosdata.jdbc.enums.SchemalessProtocolType; +import com.taosdata.jdbc.enums.SchemalessTimestampType; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +public class LineProtocolExample { + // format: measurement,tag_set field_set timestamp + private static String[] lines = { + "meters,location=Beijing.Haidian,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249000", // micro + // seconds + "meters,location=Beijing.Haidian,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611249500", + "meters,location=Beijing.Haidian,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249300", + "meters,location=Beijing.Haidian,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611249800", + }; + + private static Connection getConnection() throws SQLException { + String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata"; + return DriverManager.getConnection(jdbcUrl); + } + + private static void createDatabase(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + // the default precision is ms (microsecond), but we use us(microsecond) here. + stmt.execute("CREATE DATABASE IF NOT EXISTS test PRECISION 'us'"); + stmt.execute("USE test"); + } + } + + public static void main(String[] args) throws SQLException { + try (Connection conn = getConnection()) { + createDatabase(conn); + SchemalessWriter writer = new SchemalessWriter(conn); + writer.write(lines, SchemalessProtocolType.LINE, SchemalessTimestampType.MICRO_SECONDS); + } + } +} diff --git a/docs-examples/java/src/main/java/com/taos/example/RESTConnectExample.java b/docs-examples/java/src/main/java/com/taos/example/RESTConnectExample.java new file mode 100644 index 0000000000000000000000000000000000000000..9d077812e0ff9d0de2952d97aa12f870e6412885 --- /dev/null +++ b/docs-examples/java/src/main/java/com/taos/example/RESTConnectExample.java @@ -0,0 +1,16 @@ +package com.taos.example; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +public class RESTConnectExample { + // ANCHOR: main + public static void main(String[] args) throws SQLException { + String jdbcUrl = "jdbc:TAOS-RS://localhost:6041?user=root&password=taosdata"; + Connection conn = DriverManager.getConnection(jdbcUrl); + System.out.println("Connected"); + conn.close(); + } + // ANCHOR_END: main +} \ No newline at end of file diff --git a/docs-examples/java/src/main/java/com/taos/example/RestInsertExample.java b/docs-examples/java/src/main/java/com/taos/example/RestInsertExample.java new file mode 100644 index 0000000000000000000000000000000000000000..de89f26cbe38f9343d60aeb8d3e9ce7f67c2e764 --- /dev/null +++ b/docs-examples/java/src/main/java/com/taos/example/RestInsertExample.java @@ -0,0 +1,74 @@ +package com.taos.example; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; + + +public class RestInsertExample { + private static Connection getConnection() throws SQLException { + String jdbcUrl = "jdbc:TAOS-RS://localhost:6041?user=root&password=taosdata"; + return DriverManager.getConnection(jdbcUrl); + } + + private static List getRawData() { + return Arrays.asList( + "d1001,2018-10-03 14:38:05.000,10.30000,219,0.31000,Beijing.Chaoyang,2", + "d1001,2018-10-03 14:38:15.000,12.60000,218,0.33000,Beijing.Chaoyang,2", + "d1001,2018-10-03 14:38:16.800,12.30000,221,0.31000,Beijing.Chaoyang,2", + "d1002,2018-10-03 14:38:16.650,10.30000,218,0.25000,Beijing.Chaoyang,3", + "d1003,2018-10-03 14:38:05.500,11.80000,221,0.28000,Beijing.Haidian,2", + "d1003,2018-10-03 14:38:16.600,13.40000,223,0.29000,Beijing.Haidian,2", + "d1004,2018-10-03 14:38:05.000,10.80000,223,0.29000,Beijing.Haidian,3", + "d1004,2018-10-03 14:38:06.500,11.50000,221,0.35000,Beijing.Haidian,3" + ); + } + + + /** + * The generated SQL is: + * INSERT INTO power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES('2018-10-03 14:38:05.000',10.30000,219,0.31000) + * power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES('2018-10-03 14:38:15.000',12.60000,218,0.33000) + * power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES('2018-10-03 14:38:16.800',12.30000,221,0.31000) + * power.d1002 USING power.meters TAGS(Beijing.Chaoyang, 3) VALUES('2018-10-03 14:38:16.650',10.30000,218,0.25000) + * power.d1003 USING power.meters TAGS(Beijing.Haidian, 2) VALUES('2018-10-03 14:38:05.500',11.80000,221,0.28000) + * power.d1003 USING power.meters TAGS(Beijing.Haidian, 2) VALUES('2018-10-03 14:38:16.600',13.40000,223,0.29000) + * power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES('2018-10-03 14:38:05.000',10.80000,223,0.29000) + * power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES('2018-10-03 14:38:06.500',11.50000,221,0.35000) + */ + private static String getSQL() { + StringBuilder sb = new StringBuilder("INSERT INTO "); + for (String line : getRawData()) { + String[] ps = line.split(","); + sb.append("power." + ps[0]).append(" USING power.meters TAGS(") + .append(ps[5]).append(", ") // tag: location + .append(ps[6]) // tag: groupId + .append(") VALUES(") + .append('\'').append(ps[1]).append('\'').append(",") // ts + .append(ps[2]).append(",") // current + .append(ps[3]).append(",") // voltage + .append(ps[4]).append(") "); // phase + } + return sb.toString(); + } + + public static void insertData() throws SQLException { + try (Connection conn = getConnection()) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE DATABASE power KEEP 3650"); + stmt.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " + + "TAGS (location BINARY(64), groupId INT)"); + String sql = getSQL(); + int rowCount = stmt.executeUpdate(sql); + System.out.println("rowCount=" + rowCount); // rowCount=8 + } + } + } + + public static void main(String[] args) throws SQLException { + insertData(); + } +} diff --git a/docs-examples/java/src/main/java/com/taos/example/RestQueryExample.java b/docs-examples/java/src/main/java/com/taos/example/RestQueryExample.java new file mode 100644 index 0000000000000000000000000000000000000000..b1a1d224c6d9af2b83ac039726dcdb49a33ec2b0 --- /dev/null +++ b/docs-examples/java/src/main/java/com/taos/example/RestQueryExample.java @@ -0,0 +1,55 @@ +package com.taos.example; + +import java.sql.*; + +public class RestQueryExample { + private static Connection getConnection() throws SQLException { + String jdbcUrl = "jdbc:TAOS-RS://localhost:6041/power?user=root&password=taosdata"; + return DriverManager.getConnection(jdbcUrl); + } + + private static void printRow(ResultSet rs) throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + for (int i = 1; i <= meta.getColumnCount(); i++) { + String value = rs.getString(i); + System.out.print(value); + System.out.print("\t"); + } + System.out.println(); + } + + private static void printColName(ResultSet rs) throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + for (int i = 1; i <= meta.getColumnCount(); i++) { + String colLabel = meta.getColumnLabel(i); + System.out.print(colLabel); + System.out.print("\t"); + } + System.out.println(); + } + + private static void processResult(ResultSet rs) throws SQLException { + printColName(rs); + while (rs.next()) { + printRow(rs); + } + } + + private static void queryData() throws SQLException { + try (Connection conn = getConnection()) { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("SELECT AVG(voltage) FROM meters GROUP BY location"); + processResult(rs); + } + } + } + + public static void main(String[] args) throws SQLException { + queryData(); + } +} + +// possible output: +// avg(voltage) location +// 222.0 Beijing.Haidian +// 219.0 Beijing.Chaoyang diff --git a/docs-examples/java/src/main/java/com/taos/example/StmtInsertExample.java b/docs-examples/java/src/main/java/com/taos/example/StmtInsertExample.java new file mode 100644 index 0000000000000000000000000000000000000000..2a7ccebf41cae1a22d7516966e2c6ffb10011b64 --- /dev/null +++ b/docs-examples/java/src/main/java/com/taos/example/StmtInsertExample.java @@ -0,0 +1,84 @@ +package com.taos.example; + +import com.taosdata.jdbc.TSDBPreparedStatement; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class StmtInsertExample { + private static ArrayList tsToLongArray(String ts) { + ArrayList result = new ArrayList<>(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); + LocalDateTime localDateTime = LocalDateTime.parse(ts, formatter); + result.add(localDateTime.toInstant(ZoneOffset.of("+8")).toEpochMilli()); + return result; + } + + private static ArrayList toArray(T v) { + ArrayList result = new ArrayList<>(); + result.add(v); + return result; + } + + private static List getRawData() { + return Arrays.asList( + "d1001,2018-10-03 14:38:05.000,10.30000,219,0.31000,Beijing.Chaoyang,2", + "d1001,2018-10-03 14:38:15.000,12.60000,218,0.33000,Beijing.Chaoyang,2", + "d1001,2018-10-03 14:38:16.800,12.30000,221,0.31000,Beijing.Chaoyang,2", + "d1002,2018-10-03 14:38:16.650,10.30000,218,0.25000,Beijing.Chaoyang,3", + "d1003,2018-10-03 14:38:05.500,11.80000,221,0.28000,Beijing.Haidian,2", + "d1003,2018-10-03 14:38:16.600,13.40000,223,0.29000,Beijing.Haidian,2", + "d1004,2018-10-03 14:38:05.000,10.80000,223,0.29000,Beijing.Haidian,3", + "d1004,2018-10-03 14:38:06.500,11.50000,221,0.35000,Beijing.Haidian,3" + ); + } + + private static Connection getConnection() throws SQLException { + String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata"; + return DriverManager.getConnection(jdbcUrl); + } + + private static void createTable(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE DATABASE power KEEP 3650"); + stmt.executeUpdate("USE power"); + stmt.execute("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " + + "TAGS (location BINARY(64), groupId INT)"); + } + } + + private static void insertData() throws SQLException { + try (Connection conn = getConnection()) { + createTable(conn); + String psql = "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)"; + try (TSDBPreparedStatement pst = (TSDBPreparedStatement) conn.prepareStatement(psql)) { + for (String line : getRawData()) { + String[] ps = line.split(","); + // bind table name and tags + pst.setTableName(ps[0]); + pst.setTagString(0, ps[5]); + pst.setTagInt(1, Integer.valueOf(ps[6])); + // bind values + pst.setTimestamp(0, tsToLongArray(ps[1])); //ps[1] looks like: 2018-10-03 14:38:05.000 + pst.setFloat(1, toArray(Float.valueOf(ps[2]))); + pst.setInt(2, toArray(Integer.valueOf(ps[3]))); + pst.setFloat(3, toArray(Float.valueOf(ps[4]))); + pst.columnDataAddBatch(); + } + pst.columnDataExecuteBatch(); + } + } + } + + public static void main(String[] args) throws SQLException { + insertData(); + } +} diff --git a/docs-examples/java/src/main/java/com/taos/example/SubscribeDemo.java b/docs-examples/java/src/main/java/com/taos/example/SubscribeDemo.java new file mode 100644 index 0000000000000000000000000000000000000000..d82d03b9de561e3ea6a8e7d40a48ce9dd3f2a20d --- /dev/null +++ b/docs-examples/java/src/main/java/com/taos/example/SubscribeDemo.java @@ -0,0 +1,65 @@ +package com.taos.example; + +import com.taosdata.jdbc.TSDBConnection; +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBResultSet; +import com.taosdata.jdbc.TSDBSubscribe; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +public class SubscribeDemo { + private static final String topic = "topic-meter-current-bg-10"; + private static final String sql = "select * from meters where current > 10"; + + public static void main(String[] args) { + Connection connection = null; + TSDBSubscribe subscribe = null; + + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/power?user=root&password=taosdata"; + connection = DriverManager.getConnection(jdbcUrl, properties); + // create subscribe + subscribe = ((TSDBConnection) connection).subscribe(topic, sql, true); + int count = 0; + while (count < 10) { + // wait 1 second to avoid frequent calls to consume + TimeUnit.SECONDS.sleep(1); + // consume + TSDBResultSet resultSet = subscribe.consume(); + if (resultSet == null) { + continue; + } + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + int columnCount = metaData.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + System.out.print(metaData.getColumnLabel(i) + ": " + resultSet.getString(i) + "\t"); + } + System.out.println(); + count++; + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + if (null != subscribe) + // close subscribe + subscribe.close(true); + if (connection != null) + connection.close(); + } catch (SQLException throwable) { + throwable.printStackTrace(); + } + } + } +} \ No newline at end of file diff --git a/docs-examples/java/src/main/java/com/taos/example/TelnetLineProtocolExample.java b/docs-examples/java/src/main/java/com/taos/example/TelnetLineProtocolExample.java new file mode 100644 index 0000000000000000000000000000000000000000..1431eccf16dabaac20f60ae7e971ef49707ba509 --- /dev/null +++ b/docs-examples/java/src/main/java/com/taos/example/TelnetLineProtocolExample.java @@ -0,0 +1,45 @@ +package com.taos.example; + +import com.taosdata.jdbc.SchemalessWriter; +import com.taosdata.jdbc.enums.SchemalessProtocolType; +import com.taosdata.jdbc.enums.SchemalessTimestampType; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +public class TelnetLineProtocolExample { + // format: =[ =] + private static String[] lines = { "meters.current 1648432611249 10.3 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611250 12.6 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611249 10.8 location=Beijing.Haidian groupid=3", + "meters.current 1648432611250 11.3 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611249 219 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611250 218 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611249 221 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611250 217 location=Beijing.Haidian groupid=3", + }; + + private static Connection getConnection() throws SQLException { + String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata"; + return DriverManager.getConnection(jdbcUrl); + } + + private static void createDatabase(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + // the default precision is ms (microsecond), but we use us(microsecond) here. + stmt.execute("CREATE DATABASE IF NOT EXISTS test precision 'us'"); + stmt.execute("USE test"); + } + } + + public static void main(String[] args) throws SQLException { + try (Connection conn = getConnection()) { + createDatabase(conn); + SchemalessWriter writer = new SchemalessWriter(conn); + writer.write(lines, SchemalessProtocolType.TELNET, SchemalessTimestampType.NOT_CONFIGURED); + } + } + +} diff --git a/docs-examples/java/src/main/java/com/taos/example/WSConnectExample.java b/docs-examples/java/src/main/java/com/taos/example/WSConnectExample.java new file mode 100644 index 0000000000000000000000000000000000000000..48d1dde593cc347632efb6612946aa16e82e8b94 --- /dev/null +++ b/docs-examples/java/src/main/java/com/taos/example/WSConnectExample.java @@ -0,0 +1,21 @@ +package com.taos.example; + +import com.taosdata.jdbc.TSDBDriver; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +public class WSConnectExample { + // ANCHOR: main + public static void main(String[] args) throws SQLException { + String jdbcUrl = "jdbc:TAOS-RS://localhost:6041?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_BATCH_LOAD, "true"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + System.out.println("Connected"); + conn.close(); + } + // ANCHOR_END: main +} diff --git a/docs-examples/java/src/test/java/com/taos/test/TestAll.java b/docs-examples/java/src/test/java/com/taos/test/TestAll.java new file mode 100644 index 0000000000000000000000000000000000000000..92fe14a49d5f5ea5d7ea5f1d809867b3de0cc9d2 --- /dev/null +++ b/docs-examples/java/src/test/java/com/taos/test/TestAll.java @@ -0,0 +1,91 @@ +package com.taos.test; + +import com.taos.example.*; +import org.junit.FixMethodOrder; +import org.junit.Test; + +import java.sql.*; + +@FixMethodOrder +public class TestAll { + private String[] args = new String[]{}; + + public void dropDB(String dbName) throws SQLException { + String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata"; + try (Connection conn = DriverManager.getConnection(jdbcUrl)) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists " + dbName); + } + } + } + + public void insertData() throws SQLException { + String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata"; + try (Connection conn = DriverManager.getConnection(jdbcUrl)) { + try (Statement stmt = conn.createStatement()) { + String sql = "INSERT INTO power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES('2018-10-03 14:38:05.000',10.30000,219,0.31000)\n" + + " power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES('2018-10-03 15:38:15.000',12.60000,218,0.33000)\n" + + " power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES('2018-10-03 15:38:16.800',12.30000,221,0.31000)\n" + + " power.d1002 USING power.meters TAGS(Beijing.Chaoyang, 3) VALUES('2018-10-03 15:38:16.650',10.30000,218,0.25000)\n" + + " power.d1003 USING power.meters TAGS(Beijing.Haidian, 2) VALUES('2018-10-03 15:38:05.500',11.80000,221,0.28000)\n" + + " power.d1003 USING power.meters TAGS(Beijing.Haidian, 2) VALUES('2018-10-03 15:38:16.600',13.40000,223,0.29000)\n" + + " power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES('2018-10-03 15:38:05.000',10.80000,223,0.29000)\n" + + " power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES('2018-10-03 15:38:06.000',10.80000,223,0.29000)\n" + + " power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES('2018-10-03 15:38:07.000',10.80000,223,0.29000)\n" + + " power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES('2018-10-03 15:38:08.500',11.50000,221,0.35000)"; + + stmt.execute(sql); + } + } + } + + @Test + public void testJNIConnect() throws SQLException { + JNIConnectExample.main(args); + } + + @Test + public void testRestConnect() throws SQLException { + RESTConnectExample.main(args); + } + + @Test + public void testRestInsert() throws SQLException { + dropDB("power"); + RestInsertExample.main(args); + RestQueryExample.main(args); + } + + @Test + public void testStmtInsert() throws SQLException { + dropDB("power"); + StmtInsertExample.main(args); + } + + @Test + public void testSubscribe() { + + Thread thread = new Thread(() -> { + try { + Thread.sleep(1000); + insertData(); + } catch (SQLException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + thread.start(); + SubscribeDemo.main(args); + } + + @Test + public void testSchemaless() throws SQLException { + LineProtocolExample.main(args); + TelnetLineProtocolExample.main(args); + // for json protocol, tags may be double type. but for telnet protocol tag must be nchar type. + // To avoid type mismatch, we delete database test. + dropDB("test"); + JSONProtocolExample.main(args); + } +} diff --git a/docs-examples/node/.gitignore b/docs-examples/node/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1071ed2b9f91854f7a51424aa6c05544e803fc08 --- /dev/null +++ b/docs-examples/node/.gitignore @@ -0,0 +1,3 @@ +node_modules +package-lock.json +yarn-lock.json \ No newline at end of file diff --git a/docs-examples/node/nativeexample/async_query_example.js b/docs-examples/node/nativeexample/async_query_example.js new file mode 100644 index 0000000000000000000000000000000000000000..25b78bc48a82454cc0f9db69b77323a00f3852b4 --- /dev/null +++ b/docs-examples/node/nativeexample/async_query_example.js @@ -0,0 +1,21 @@ +const taos = require("td2.0-connector"); +const conn = taos.connect({ host: "localhost", database: "power" }); +const cursor = conn.cursor(); + +function queryExample() { + cursor + .query("SELECT ts, current FROM meters LIMIT 2") + .execute_a() + .then((result) => { + result.pretty(); + }); +} + +try { + queryExample(); +} finally { + setTimeout(() => { + conn.close(); + }, 2000); +} +// bug here: jira 14506 diff --git a/docs-examples/node/nativeexample/connect.js b/docs-examples/node/nativeexample/connect.js new file mode 100644 index 0000000000000000000000000000000000000000..da791c566e3aa40315b6f35cfeca704040cd8107 --- /dev/null +++ b/docs-examples/node/nativeexample/connect.js @@ -0,0 +1,13 @@ +const taos = require("td2.0-connector"); + +var conn = taos.connect({ + host: "localhost", + port: 6030, + user: "root", + password: "taosdata", +}); +conn.close(); + +// run with: node connect.js +// output: +// Successfully connected to TDengine diff --git a/docs-examples/node/nativeexample/influxdb_line_example.js b/docs-examples/node/nativeexample/influxdb_line_example.js new file mode 100644 index 0000000000000000000000000000000000000000..a9fc6d11df0b335b92bb3292baaa017cb4bc42ea --- /dev/null +++ b/docs-examples/node/nativeexample/influxdb_line_example.js @@ -0,0 +1,34 @@ +const taos = require("td2.0-connector"); + +const conn = taos.connect({ + host: "localhost", +}); + +const cursor = conn.cursor(); + +function createDatabase() { + cursor.execute("CREATE DATABASE test"); + cursor.execute("USE test"); +} + +function insertData() { + const lines = [ + "meters,location=Beijing.Haidian,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249", + "meters,location=Beijing.Haidian,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250", + "meters,location=Beijing.Haidian,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249", + "meters,location=Beijing.Haidian,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250", + ]; + cursor.schemalessInsert( + lines, + taos.SCHEMALESS_PROTOCOL.TSDB_SML_LINE_PROTOCOL, + taos.SCHEMALESS_PRECISION.TSDB_SML_TIMESTAMP_MILLI_SECONDS + ); +} + +try { + createDatabase(); + insertData(); +} finally { + cursor.close(); + conn.close(); +} diff --git a/docs-examples/node/nativeexample/insert_example.js b/docs-examples/node/nativeexample/insert_example.js new file mode 100644 index 0000000000000000000000000000000000000000..85a353f889176655654d8c39c9a905054d3b6622 --- /dev/null +++ b/docs-examples/node/nativeexample/insert_example.js @@ -0,0 +1,31 @@ +const taos = require("td2.0-connector"); + +const conn = taos.connect({ + host: "localhost", +}); + +const cursor = conn.cursor(); +try { + cursor.execute("CREATE DATABASE power"); + cursor.execute("USE power"); + cursor.execute( + "CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)" + ); + var sql = `INSERT INTO power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) +power.d1002 USING power.meters TAGS(Beijing.Chaoyang, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) +power.d1003 USING power.meters TAGS(Beijing.Haidian, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) +power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)`; + cursor.execute(sql); +} finally { + cursor.close(); + conn.close(); +} + +// run with: node insert_example.js +// output: +// Successfully connected to TDengine +// Query OK, 0 row(s) affected (0.00509570s) +// Query OK, 0 row(s) affected (0.00130880s) +// Query OK, 0 row(s) affected (0.00467900s) +// Query OK, 8 row(s) affected (0.04043550s) +// Connection is closed diff --git a/docs-examples/node/nativeexample/multi_bind_example.js b/docs-examples/node/nativeexample/multi_bind_example.js new file mode 100644 index 0000000000000000000000000000000000000000..d52581ec8e10c6edfbc8fc8f7ca78512b5c93d74 --- /dev/null +++ b/docs-examples/node/nativeexample/multi_bind_example.js @@ -0,0 +1,53 @@ +const taos = require("td2.0-connector"); + +const conn = taos.connect({ + host: "localhost", +}); + +const cursor = conn.cursor(); + +function prepareSTable() { + cursor.execute("CREATE DATABASE power"); + cursor.execute("USE power"); + cursor.execute( + "CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)" + ); +} + +//ANCHOR: insertData +function insertData() { + // init + cursor.stmtInit(); + // prepare + cursor.stmtPrepare( + "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)" + ); + + // bind table name and tags + let tagBind = new taos.TaosBind(2); + tagBind.bindBinary("Beijing.Chaoyang"); + tagBind.bindInt(2); + cursor.stmtSetTbnameTags("d1001", tagBind.getBind()); + + // bind values + let valueBind = new taos.TaosMultiBindArr(4); + valueBind.multiBindTimestamp([1648432611249, 1648432611749]); + valueBind.multiBindFloat([10.3, 12.6]); + valueBind.multiBindInt([219, 218]); + valueBind.multiBindFloat([0.31, 0.33]); + cursor.stmtBindParamBatch(valueBind.getMultiBindArr()); + cursor.stmtAddBatch(); + + // execute + cursor.stmtExecute(); + cursor.stmtClose(); +} +//ANCHOR_END: insertData + +try { + prepareSTable(); + insertData(); +} finally { + cursor.close(); + conn.close(); +} diff --git a/docs-examples/node/nativeexample/opentsdb_json_example.js b/docs-examples/node/nativeexample/opentsdb_json_example.js new file mode 100644 index 0000000000000000000000000000000000000000..6d436a8e9ebe0230bba22064e8fb6c180c14b5d1 --- /dev/null +++ b/docs-examples/node/nativeexample/opentsdb_json_example.js @@ -0,0 +1,55 @@ +const taos = require("td2.0-connector"); + +const conn = taos.connect({ + host: "localhost", +}); + +const cursor = conn.cursor(); + +function createDatabase() { + cursor.execute("CREATE DATABASE test"); + cursor.execute("USE test"); +} + +function insertData() { + const lines = [ + { + metric: "meters.current", + timestamp: 1648432611249, + value: 10.3, + tags: { location: "Beijing.Chaoyang", groupid: 2 }, + }, + { + metric: "meters.voltage", + timestamp: 1648432611249, + value: 219, + tags: { location: "Beijing.Haidian", groupid: 1 }, + }, + { + metric: "meters.current", + timestamp: 1648432611250, + value: 12.6, + tags: { location: "Beijing.Chaoyang", groupid: 2 }, + }, + { + metric: "meters.voltage", + timestamp: 1648432611250, + value: 221, + tags: { location: "Beijing.Haidian", groupid: 1 }, + }, + ]; + + cursor.schemalessInsert( + [JSON.stringify(lines)], + taos.SCHEMALESS_PROTOCOL.TSDB_SML_JSON_PROTOCOL, + taos.SCHEMALESS_PRECISION.TSDB_SML_TIMESTAMP_NOT_CONFIGURED + ); +} + +try { + createDatabase(); + insertData(); +} finally { + cursor.close(); + conn.close(); +} diff --git a/docs-examples/node/nativeexample/opentsdb_telnet_example.js b/docs-examples/node/nativeexample/opentsdb_telnet_example.js new file mode 100644 index 0000000000000000000000000000000000000000..01e79c2dcacd923cd708d1d228959a628d0ff26a --- /dev/null +++ b/docs-examples/node/nativeexample/opentsdb_telnet_example.js @@ -0,0 +1,38 @@ +const taos = require("td2.0-connector"); + +const conn = taos.connect({ + host: "localhost", +}); + +const cursor = conn.cursor(); + +function createDatabase() { + cursor.execute("CREATE DATABASE test"); + cursor.execute("USE test"); +} + +function insertData() { + const lines = [ + "meters.current 1648432611249 10.3 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611250 12.6 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611249 10.8 location=Beijing.Haidian groupid=3", + "meters.current 1648432611250 11.3 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611249 219 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611250 218 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611249 221 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611250 217 location=Beijing.Haidian groupid=3", + ]; + cursor.schemalessInsert( + lines, + taos.SCHEMALESS_PROTOCOL.TSDB_SML_TELNET_PROTOCOL, + taos.SCHEMALESS_PRECISION.TSDB_SML_TIMESTAMP_NOT_CONFIGURED + ); +} + +try { + createDatabase(); + insertData(); +} finally { + cursor.close(); + conn.close(); +} diff --git a/docs-examples/node/nativeexample/param_bind_example.js b/docs-examples/node/nativeexample/param_bind_example.js new file mode 100644 index 0000000000000000000000000000000000000000..9117f46c3eeabd9009b72fa9d4a8503e65884242 --- /dev/null +++ b/docs-examples/node/nativeexample/param_bind_example.js @@ -0,0 +1,57 @@ +const taos = require("td2.0-connector"); + +const conn = taos.connect({ + host: "localhost", +}); + +const cursor = conn.cursor(); + +function prepareSTable() { + cursor.execute("CREATE DATABASE power"); + cursor.execute("USE power"); + cursor.execute( + "CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)" + ); +} + +function insertData() { + // init + cursor.stmtInit(); + // prepare + cursor.stmtPrepare( + "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)" + ); + + // bind table name and tags + let tagBind = new taos.TaosBind(2); + tagBind.bindBinary("Beijing.Chaoyang"); + tagBind.bindInt(2); + cursor.stmtSetTbnameTags("d1001", tagBind.getBind()); + + // bind values + let rows = [ + [1648432611249, 10.3, 219, 0.31], + [1648432611749, 12.6, 218, 0.33], + ]; + for (let row of rows) { + let valueBind = new taos.TaosBind(4); + valueBind.bindTimestamp(row[0]); + valueBind.bindFloat(row[1]); + valueBind.bindInt(row[2]); + valueBind.bindFloat(row[3]); + cursor.stmtBindParam(valueBind.getBind()); + cursor.stmtAddBatch(); + } + + // execute + cursor.stmtExecute(); + cursor.stmtClose(); +} + +try { + prepareSTable(); + insertData(); +} finally { + cursor.close(); + conn.close(); +} diff --git a/docs-examples/node/nativeexample/query_example.js b/docs-examples/node/nativeexample/query_example.js new file mode 100644 index 0000000000000000000000000000000000000000..a51bc939ae8dc458f019ac3bf006d9320dfdee81 --- /dev/null +++ b/docs-examples/node/nativeexample/query_example.js @@ -0,0 +1,17 @@ +const taos = require("td2.0-connector"); + +const conn = taos.connect({ host: "localhost", database: "power" }); +const cursor = conn.cursor(); +const query = cursor.query("SELECT ts, current FROM meters LIMIT 2"); +query.execute().then(function (result) { + result.pretty(); +}); + +// output: +// Successfully connected to TDengine +// Query OK, 2 row(s) in set (0.00317767s) + +// ts | current | +// ======================================================= +// 2018-10-03 14:38:05.000 | 10.3 | +// 2018-10-03 14:38:15.000 | 12.6 | diff --git a/docs-examples/node/nativeexample/subscribe_demo.js b/docs-examples/node/nativeexample/subscribe_demo.js new file mode 100644 index 0000000000000000000000000000000000000000..bdf0c98687c359ba20664e869a25690dffbabd29 --- /dev/null +++ b/docs-examples/node/nativeexample/subscribe_demo.js @@ -0,0 +1,4 @@ +const taos = require("td2.0-connector"); + +const conn = taos.connect({ host: "localhost", database: "power" }); +// 未完成 \ No newline at end of file diff --git a/docs-examples/node/package.json b/docs-examples/node/package.json new file mode 100644 index 0000000000000000000000000000000000000000..f56196d2e5011b1c56e59d78d2ce5fdda98205e9 --- /dev/null +++ b/docs-examples/node/package.json @@ -0,0 +1,10 @@ +{ + "name": "examples", + "version": "1.0.0", + "main": "index.js", + "license": "MIT", + "dependencies": { + "td2.0-connector": "^2.0.12", + "td2.0-rest-connector": "^1.0.0" + } +} diff --git a/docs-examples/node/restexample/connect.js b/docs-examples/node/restexample/connect.js new file mode 100644 index 0000000000000000000000000000000000000000..b84ce2fadf9177983701582b05200986ea9d5ba7 --- /dev/null +++ b/docs-examples/node/restexample/connect.js @@ -0,0 +1,20 @@ +const { options, connect } = require("td2.0-rest-connector"); + +async function test() { + options.path = "/rest/sqlt"; + options.host = "localhost"; + let conn = connect(options); + let cursor = conn.cursor(); + try { + let res = await cursor.query("SELECT server_version()"); + res.toString(); + } catch (err) { + console.log(err); + } +} +test(); + +// output: +// server_version() | +// =================== +// 2.4.0.12 | diff --git a/docs-examples/php/connect.php b/docs-examples/php/connect.php new file mode 100644 index 0000000000000000000000000000000000000000..5af77b9768e5c5ac4b774b433479a4ac8902beda --- /dev/null +++ b/docs-examples/php/connect.php @@ -0,0 +1,20 @@ +connect(); +} catch (TDengineException $e) { + // 连接失败捕获异常 + throw $e; +} diff --git a/docs-examples/php/insert.php b/docs-examples/php/insert.php new file mode 100644 index 0000000000000000000000000000000000000000..0d9cfc4843a2ec3e72d0ad128fa4c2650d6b9cf6 --- /dev/null +++ b/docs-examples/php/insert.php @@ -0,0 +1,33 @@ +connect(); + + // 插入 + $connection->query('CREATE DATABASE if not exists power'); + $connection->query('CREATE STABLE if not exists meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)'); + $resource = $connection->query(<<<'SQL' + INSERT INTO power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS(Beijing.Chaoyang, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS(Beijing.Haidian, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000) + SQL); + + // 影响行数 + var_dump($resource->affectedRows()); +} catch (TDengineException $e) { + // 捕获异常 + throw $e; +} diff --git a/docs-examples/php/insert_stmt.php b/docs-examples/php/insert_stmt.php new file mode 100644 index 0000000000000000000000000000000000000000..5d4b4809d215d781807c21172982feff2171fe07 --- /dev/null +++ b/docs-examples/php/insert_stmt.php @@ -0,0 +1,49 @@ +connect(); + + // 插入 + $connection->query('CREATE DATABASE if not exists power'); + $connection->query('CREATE STABLE if not exists meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)'); + $stmt = $connection->prepare('INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)'); + + // 设置表名和标签 + $stmt->setTableNameTags('d1001', [ + // 支持格式同参数绑定 + [TDengine\TSDB_DATA_TYPE_BINARY, 'Beijing.Chaoyang'], + [TDengine\TSDB_DATA_TYPE_INT, 2], + ]); + + $stmt->bindParams([ + [TDengine\TSDB_DATA_TYPE_TIMESTAMP, 1648432611249], + [TDengine\TSDB_DATA_TYPE_FLOAT, 10.3], + [TDengine\TSDB_DATA_TYPE_INT, 219], + [TDengine\TSDB_DATA_TYPE_FLOAT, 0.31], + ]); + $stmt->bindParams([ + [TDengine\TSDB_DATA_TYPE_TIMESTAMP, 1648432611749], + [TDengine\TSDB_DATA_TYPE_FLOAT, 12.6], + [TDengine\TSDB_DATA_TYPE_INT, 218], + [TDengine\TSDB_DATA_TYPE_FLOAT, 0.33], + ]); + $resource = $stmt->execute(); + + // 影响行数 + var_dump($resource->affectedRows()); +} catch (TDengineException $e) { + // 捕获异常 + throw $e; +} diff --git a/docs-examples/php/query.php b/docs-examples/php/query.php new file mode 100644 index 0000000000000000000000000000000000000000..4e86a2cec7426887686049977a8647e786ac2744 --- /dev/null +++ b/docs-examples/php/query.php @@ -0,0 +1,23 @@ +connect(); + + $resource = $connection->query('SELECT ts, current FROM meters LIMIT 2'); + var_dump($resource->fetch()); +} catch (TDengineException $e) { + // 捕获异常 + throw $e; +} diff --git a/docs-examples/python/.gitignore b/docs-examples/python/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..723ef36f4e4f32c4560383aa5987c575a30c6535 --- /dev/null +++ b/docs-examples/python/.gitignore @@ -0,0 +1 @@ +.idea \ No newline at end of file diff --git a/docs-examples/python/.gitkeep b/docs-examples/python/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/docs-examples/python/async_query_example.py b/docs-examples/python/async_query_example.py new file mode 100644 index 0000000000000000000000000000000000000000..a78f44230a34d40a5326dea367bbad11a6e51da2 --- /dev/null +++ b/docs-examples/python/async_query_example.py @@ -0,0 +1,72 @@ +import time +from ctypes import * + +from taos import * + + +def fetch_callback(p_param, p_result, num_of_rows): + print("fetched ", num_of_rows, "rows") + p = cast(p_param, POINTER(Counter)) + result = TaosResult(p_result) + + if num_of_rows == 0: + print("fetching completed") + p.contents.done = True + result.close() + return + if num_of_rows < 0: + p.contents.done = True + result.check_error(num_of_rows) + result.close() + return None + + for row in result.rows_iter(num_of_rows): + print(row) + p.contents.count += result.row_count + result.fetch_rows_a(fetch_callback, p_param) + + +def query_callback(p_param, p_result, code): + if p_result is None: + return + result = TaosResult(p_result) + if code == 0: + result.fetch_rows_a(fetch_callback, p_param) + result.check_error(code) + + +class Counter(Structure): + _fields_ = [("count", c_int), ("done", c_bool)] + + def __str__(self): + return "{ count: %d, done: %s }" % (self.count, self.done) + + +def test_query(conn): + counter = Counter(count=0) + conn.query_a("select ts, current, voltage from power.meters", query_callback, byref(counter)) + + while not counter.done: + print(counter) + time.sleep(1) + print(counter) + conn.close() + + +if __name__ == "__main__": + test_query(connect()) + +# possible output: +# { count: 0, done: False } +# fetched 8 rows +# 1538548685000 10.300000 219 +# 1538548695000 12.600000 218 +# 1538548696800 12.300000 221 +# 1538548696650 10.300000 218 +# 1538548685500 11.800000 221 +# 1538548696600 13.400000 223 +# 1538548685500 10.800000 223 +# 1538548686500 11.500000 221 +# fetched 0 rows +# fetching completed +# { count: 8, done: True } diff --git a/docs-examples/python/bind_param_example.py b/docs-examples/python/bind_param_example.py new file mode 100644 index 0000000000000000000000000000000000000000..503a2eb5dd91a3516f87a4d3c1c3218cb6505236 --- /dev/null +++ b/docs-examples/python/bind_param_example.py @@ -0,0 +1,60 @@ +import taos +from datetime import datetime + +# note: lines have already been sorted by table name +lines = [('d1001', '2018-10-03 14:38:05.000', 10.30000, 219, 0.31000, 'Beijing.Chaoyang', 2), + ('d1001', '2018-10-03 14:38:15.000', 12.60000, 218, 0.33000, 'Beijing.Chaoyang', 2), + ('d1001', '2018-10-03 14:38:16.800', 12.30000, 221, 0.31000, 'Beijing.Chaoyang', 2), + ('d1002', '2018-10-03 14:38:16.650', 10.30000, 218, 0.25000, 'Beijing.Chaoyang', 3), + ('d1003', '2018-10-03 14:38:05.500', 11.80000, 221, 0.28000, 'Beijing.Haidian', 2), + ('d1003', '2018-10-03 14:38:16.600', 13.40000, 223, 0.29000, 'Beijing.Haidian', 2), + ('d1004', '2018-10-03 14:38:05.000', 10.80000, 223, 0.29000, 'Beijing.Haidian', 3), + ('d1004', '2018-10-03 14:38:06.500', 11.50000, 221, 0.35000, 'Beijing.Haidian', 3)] + + +def get_ts(ts: str): + dt = datetime.strptime(ts, '%Y-%m-%d %H:%M:%S.%f') + return int(dt.timestamp() * 1000) + + +def create_stable(): + conn = taos.connect() + try: + conn.execute("CREATE DATABASE power") + conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " + "TAGS (location BINARY(64), groupId INT)") + finally: + conn.close() + + +def bind_row_by_row(stmt: taos.TaosStmt): + tb_name = None + for row in lines: + if tb_name != row[0]: + tb_name = row[0] + tags: taos.TaosBind = taos.new_bind_params(2) # 2 is count of tags + tags[0].binary(row[5]) # location + tags[1].int(row[6]) # groupId + stmt.set_tbname_tags(tb_name, tags) + values: taos.TaosBind = taos.new_bind_params(4) # 4 is count of columns + values[0].timestamp(get_ts(row[1])) + values[1].float(row[2]) + values[2].int(row[3]) + values[3].float(row[4]) + stmt.bind_param(values) + + +def insert_data(): + conn = taos.connect(database="power") + try: + stmt = conn.statement("INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)") + bind_row_by_row(stmt) + stmt.execute() + stmt.close() + finally: + conn.close() + + +if __name__ == '__main__': + create_stable() + insert_data() diff --git a/docs-examples/python/conn_native_pandas.py b/docs-examples/python/conn_native_pandas.py new file mode 100644 index 0000000000000000000000000000000000000000..314759f7662c7bf4c9df2c8b3396ad3101c91cd4 --- /dev/null +++ b/docs-examples/python/conn_native_pandas.py @@ -0,0 +1,19 @@ +import pandas +from sqlalchemy import create_engine + +engine = create_engine("taos://root:taosdata@localhost:6030/power") +df = pandas.read_sql("SELECT * FROM meters", engine) + +# print index +print(df.index) +# print data type of element in ts column +print(type(df.ts[0])) +print(df.head(3)) + +# output: +# RangeIndex(start=0, stop=8, step=1) +# +# ts current voltage phase location groupid +# 0 2018-10-03 14:38:05.000 10.3 219 0.31 beijing.chaoyang 2 +# 1 2018-10-03 14:38:15.000 12.6 218 0.33 beijing.chaoyang 2 +# 2 2018-10-03 14:38:16.800 12.3 221 0.31 beijing.chaoyang 2 diff --git a/docs-examples/python/conn_rest_pandas.py b/docs-examples/python/conn_rest_pandas.py new file mode 100644 index 0000000000000000000000000000000000000000..143e4275fa4eda685766297e4b90cba3935a574d --- /dev/null +++ b/docs-examples/python/conn_rest_pandas.py @@ -0,0 +1,19 @@ +import pandas +from sqlalchemy import create_engine + +engine = create_engine("taosrest://root:taosdata@localhost:6041") +df: pandas.DataFrame = pandas.read_sql("SELECT * FROM power.meters", engine) + +# print index +print(df.index) +# print data type of element in ts column +print(type(df.ts[0])) +print(df.head(3)) + +# output: +# +# RangeIndex(start=0, stop=8, step=1) +# ts current ... location groupid +# 0 2018-10-03 14:38:05+08:00 10.3 ... beijing.chaoyang 2 +# 1 2018-10-03 14:38:15+08:00 12.6 ... beijing.chaoyang 2 +# 2 2018-10-03 14:38:16.800000+08:00 12.3 ... beijing.chaoyang 2 diff --git a/docs-examples/python/connect_example.py b/docs-examples/python/connect_example.py new file mode 100644 index 0000000000000000000000000000000000000000..719af79fd12c14cc16e4e3e3c4d6362da5a7520d --- /dev/null +++ b/docs-examples/python/connect_example.py @@ -0,0 +1,19 @@ +import taos + + +def test_connection(): + # all parameters are optional. + # if database is specified, + # then it must exist. + conn = taos.connect(host="localhost", + port=6030, + user="root", + password="taosdata", + database="log") + print('client info:', conn.client_info) + print('server info:', conn.server_info) + conn.close() + + +if __name__ == "__main__": + test_connection() diff --git a/docs-examples/python/connect_native_reference.py b/docs-examples/python/connect_native_reference.py new file mode 100644 index 0000000000000000000000000000000000000000..c17e9795b58724f6646b8d7c0f84047098a93d69 --- /dev/null +++ b/docs-examples/python/connect_native_reference.py @@ -0,0 +1,20 @@ +import taos + +conn: taos.TaosConnection = taos.connect(host="localhost", + user="root", + password="taosdata", + database="test", + port=6030, + config="/etc/taos", # for windows the default value is C:\TDengine\cfg + timezone="Asia/Shanghai") # default your host's timezone + +server_version = conn.server_info +print("server_version", server_version) +client_version = conn.client_info +print("client_version", client_version) # 2.4.0.16 + +conn.close() + +# possible output: +# 2.4.0.16 +# 2.4.0.16 diff --git a/docs-examples/python/connect_rest_examples.py b/docs-examples/python/connect_rest_examples.py new file mode 100644 index 0000000000000000000000000000000000000000..a043d506b965bc31179dbb6f38749d196ab338ff --- /dev/null +++ b/docs-examples/python/connect_rest_examples.py @@ -0,0 +1,45 @@ +# ANCHOR: connect +from taosrest import connect, TaosRestConnection, TaosRestCursor + +conn: TaosRestConnection = connect(host="localhost", + user="root", + password="taosdata", + port=6041, + timeout=30) + +# ANCHOR_END: connect +# ANCHOR: basic +# create STable +cursor: TaosRestCursor = conn.cursor() +cursor.execute("DROP DATABASE IF EXISTS power") +cursor.execute("CREATE DATABASE power") +cursor.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") + +# insert data +cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS(Beijing.Chaoyang, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS(Beijing.Haidian, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""") +print("inserted row count:", cursor.rowcount) + +# query data +cursor.execute("SELECT * FROM power.meters LIMIT 3") +# get total rows +print("queried row count:", cursor.rowcount) +# get column names from cursor +column_names = [meta[0] for meta in cursor.description] +# get rows +data: list[tuple] = cursor.fetchall() +print(column_names) +for row in data: + print(row) + +# output: +# inserted row count: 8 +# queried row count: 3 +# ['ts', 'current', 'voltage', 'phase', 'location', 'groupid'] +# [datetime.datetime(2018, 10, 3, 14, 38, 5, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 10.3, 219, 0.31, 'beijing.chaoyang', 2] +# [datetime.datetime(2018, 10, 3, 14, 38, 15, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 12.6, 218, 0.33, 'beijing.chaoyang', 2] +# [datetime.datetime(2018, 10, 3, 14, 38, 16, 800000, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 12.3, 221, 0.31, 'beijing.chaoyang', 2] + +# ANCHOR_END: basic diff --git a/docs-examples/python/connection_usage_native_reference.py b/docs-examples/python/connection_usage_native_reference.py new file mode 100644 index 0000000000000000000000000000000000000000..4803511e427bf4d906fd3a14ff6faf5a000da96c --- /dev/null +++ b/docs-examples/python/connection_usage_native_reference.py @@ -0,0 +1,45 @@ +import taos + +# ANCHOR: insert +conn = taos.connect() +# Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. +conn.execute("DROP DATABASE IF EXISTS test") +conn.execute("CREATE DATABASE test") +# change database. same as execute "USE db" +conn.select_db("test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") +affected_row: int = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m 24.4)") +print("affected_row", affected_row) +# output: +# affected_row 3 +# ANCHOR_END: insert + +# ANCHOR: query +# Execute a sql and get its result set. It's useful for SELECT statement +result: taos.TaosResult = conn.query("SELECT * from weather") + +# Get fields from result +fields: taos.field.TaosFields = result.fields +for field in fields: + print(field) # {name: ts, type: 9, bytes: 8} + +# output: +# {name: ts, type: 9, bytes: 8} +# {name: temperature, type: 6, bytes: 4} +# {name: location, type: 4, bytes: 4} + +# Get data from result as list of tuple +data = result.fetch_all() +print(data) +# output: +# [(datetime.datetime(2022, 4, 27, 9, 4, 25, 367000), 23.5, 1), (datetime.datetime(2022, 4, 27, 9, 5, 25, 367000), 23.5, 1), (datetime.datetime(2022, 4, 27, 9, 6, 25, 367000), 24.399999618530273, 1)] + +# Or get data from result as a list of dict +# map_data = result.fetch_all_into_dict() +# print(map_data) +# output: +# [{'ts': datetime.datetime(2022, 4, 27, 9, 1, 15, 343000), 'temperature': 23.5, 'location': 1}, {'ts': datetime.datetime(2022, 4, 27, 9, 2, 15, 343000), 'temperature': 23.5, 'location': 1}, {'ts': datetime.datetime(2022, 4, 27, 9, 3, 15, 343000), 'temperature': 24.399999618530273, 'location': 1}] +# ANCHOR_END: query + + +conn.close() diff --git a/docs-examples/python/cursor_usage_native_reference.py b/docs-examples/python/cursor_usage_native_reference.py new file mode 100644 index 0000000000000000000000000000000000000000..a5103810f0199708f94b5c36e1dc080b7998e420 --- /dev/null +++ b/docs-examples/python/cursor_usage_native_reference.py @@ -0,0 +1,32 @@ +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("DROP DATABASE IF EXISTS test") +cursor.execute("CREATE DATABASE test") +cursor.execute("USE test") +cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") + +for i in range(1000): + location = str(i % 10) + tb = "t" + location + cursor.execute(f"INSERT INTO {tb} USING weather TAGS({location}) VALUES (now+{i}a, 23.5) (now+{i + 1}a, 23.5)") + +cursor.execute("SELECT count(*) FROM weather") +data = cursor.fetchall() +print("count:", data[0][0]) +cursor.execute("SELECT tbname, * FROM weather LIMIT 2") +col_names = [meta[0] for meta in cursor.description] +print(col_names) +rows = cursor.fetchall() +print(rows) + +cursor.close() +conn.close() + +# output: +# count: 2000 +# ['tbname', 'ts', 'temperature', 'location'] +# row_count: -1 +# [('t0', datetime.datetime(2022, 4, 27, 14, 54, 24, 392000), 23.5, 0), ('t0', datetime.datetime(2022, 4, 27, 14, 54, 24, 393000), 23.5, 0)] diff --git a/docs-examples/python/handle_exception.py b/docs-examples/python/handle_exception.py new file mode 100644 index 0000000000000000000000000000000000000000..59554465c0891c80f0507656fcc79538c3811fa7 --- /dev/null +++ b/docs-examples/python/handle_exception.py @@ -0,0 +1,19 @@ +import taos + +try: + conn = taos.connect() + conn.execute("CREATE TABLE 123") # wrong sql +except taos.Error as e: + print(e) + print("exception class: ", e.__class__.__name__) + print("error number:", e.errno) + print("error message:", e.msg) +except BaseException as other: + print("exception occur") + print(other) + +# output: +# [0x0216]: syntax error near 'Incomplete SQL statement' +# exception class: ProgrammingError +# error number: -2147483114 +# error message: syntax error near 'Incomplete SQL statement' diff --git a/docs-examples/python/json_protocol_example.py b/docs-examples/python/json_protocol_example.py new file mode 100644 index 0000000000000000000000000000000000000000..5bb4d629bccf3d79e74b381d6259de86d6522315 --- /dev/null +++ b/docs-examples/python/json_protocol_example.py @@ -0,0 +1,38 @@ +import json + +import taos +from taos import SmlProtocol, SmlPrecision + +lines = [{"metric": "meters.current", "timestamp": 1648432611249, "value": 10.3, "tags": {"location": "Beijing.Chaoyang", "groupid": 2}}, + {"metric": "meters.voltage", "timestamp": 1648432611249, "value": 219, + "tags": {"location": "Beijing.Haidian", "groupid": 1}}, + {"metric": "meters.current", "timestamp": 1648432611250, "value": 12.6, + "tags": {"location": "Beijing.Chaoyang", "groupid": 2}}, + {"metric": "meters.voltage", "timestamp": 1648432611250, "value": 221, "tags": {"location": "Beijing.Haidian", "groupid": 1}}] + + +def get_connection(): + return taos.connect() + + +def create_database(conn): + conn.execute("CREATE DATABASE test") + conn.execute("USE test") + + +def insert_lines(conn): + global lines + lines = json.dumps(lines) + # note: the first parameter must be a list with only one element. + affected_rows = conn.schemaless_insert( + [lines], SmlProtocol.JSON_PROTOCOL, SmlPrecision.NOT_CONFIGURED) + print(affected_rows) # 4 + + +if __name__ == '__main__': + connection = get_connection() + try: + create_database(connection) + insert_lines(connection) + finally: + connection.close() diff --git a/docs-examples/python/line_protocol_example.py b/docs-examples/python/line_protocol_example.py new file mode 100644 index 0000000000000000000000000000000000000000..02baeb2104f9f48984b4d34afb5e67af641d4e32 --- /dev/null +++ b/docs-examples/python/line_protocol_example.py @@ -0,0 +1,34 @@ +import taos +from taos import SmlProtocol, SmlPrecision + +lines = ["meters,location=Beijing.Haidian,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249000", + "meters,location=Beijing.Haidian,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611249500", + "meters,location=Beijing.Haidian,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249300", + "meters,location=Beijing.Haidian,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611249800", + ] + + +def get_connection(): + # create connection use firstEP in taos.cfg. + return taos.connect() + + +def create_database(conn): + # the default precision is ms (microsecond), but we use us(microsecond) here. + conn.execute("CREATE DATABASE test precision 'us'") + conn.execute("USE test") + + +def insert_lines(conn): + affected_rows = conn.schemaless_insert( + lines, SmlProtocol.LINE_PROTOCOL, SmlPrecision.MICRO_SECONDS) + print(affected_rows) # 8 + + +if __name__ == '__main__': + connection = get_connection() + try: + create_database(connection) + insert_lines(connection) + finally: + connection.close() diff --git a/docs-examples/python/multi_bind_example.py b/docs-examples/python/multi_bind_example.py new file mode 100644 index 0000000000000000000000000000000000000000..1714121d72705ab8d619a41f3463af4aa3193871 --- /dev/null +++ b/docs-examples/python/multi_bind_example.py @@ -0,0 +1,88 @@ +import taos +from datetime import datetime + +# ANCHOR: bind_batch +table_tags = { + "d1001": ('Beijing.Chaoyang', 2), + "d1002": ('Beijing.Chaoyang', 3), + "d1003": ('Beijing.Haidian', 2), + "d1004": ('Beijing.Haidian', 3) +} + +table_values = { + "d1001": [ + ['2018-10-03 14:38:05.000', '2018-10-03 14:38:15.000', '2018-10-03 14:38:16.800'], + [10.3, 12.6, 12.3], + [219, 218, 221], + [0.31, 0.33, 0.32] + ], + "d1002": [ + ['2018-10-03 14:38:16.650'], [10.3], [218], [0.25] + ], + "d1003": [ + ['2018-10-03 14:38:05.500', '2018-10-03 14:38:16.600'], + [11.8, 13.4], + [221, 223], + [0.28, 0.29] + ], + "d1004": [ + ['2018-10-03 14:38:05.500', '2018-10-03 14:38:06.500'], + [10.8, 11.5], + [223, 221], + [0.29, 0.35] + ] +} + + +def bind_multi_rows(stmt: taos.TaosStmt): + """ + batch bind example + """ + for tb_name in table_values.keys(): + tags = table_tags[tb_name] + tag_params = taos.new_bind_params(2) + tag_params[0].binary(tags[0]) + tag_params[1].int(tags[1]) + stmt.set_tbname_tags(tb_name, tag_params) + + values = table_values[tb_name] + value_params = taos.new_multi_binds(4) + value_params[0].timestamp([get_ts(t) for t in values[0]]) + value_params[1].float(values[1]) + value_params[2].int(values[2]) + value_params[3].float(values[3]) + stmt.bind_param_batch(value_params) + + +def insert_data(): + conn = taos.connect(database="power") + try: + stmt = conn.statement("INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)") + bind_multi_rows(stmt) + stmt.execute() + stmt.close() + finally: + conn.close() + + +# ANCHOR_END: bind_batch + + +def create_stable(): + conn = taos.connect() + try: + conn.execute("CREATE DATABASE power") + conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " + "TAGS (location BINARY(64), groupId INT)") + finally: + conn.close() + + +def get_ts(ts: str): + dt = datetime.strptime(ts, '%Y-%m-%d %H:%M:%S.%f') + return int(dt.timestamp() * 1000) + + +if __name__ == '__main__': + create_stable() + insert_data() diff --git a/docs-examples/python/native_insert_example.py b/docs-examples/python/native_insert_example.py new file mode 100644 index 0000000000000000000000000000000000000000..94d4888a8f5330b9e39d5ae051fcb68f9825505f --- /dev/null +++ b/docs-examples/python/native_insert_example.py @@ -0,0 +1,60 @@ +import taos + +lines = ["d1001,2018-10-03 14:38:05.000,10.30000,219,0.31000,Beijing.Chaoyang,2", + "d1004,2018-10-03 14:38:05.000,10.80000,223,0.29000,Beijing.Haidian,3", + "d1003,2018-10-03 14:38:05.500,11.80000,221,0.28000,Beijing.Haidian,2", + "d1004,2018-10-03 14:38:06.500,11.50000,221,0.35000,Beijing.Haidian,3", + "d1002,2018-10-03 14:38:16.650,10.30000,218,0.25000,Beijing.Chaoyang,3", + "d1001,2018-10-03 14:38:15.000,12.60000,218,0.33000,Beijing.Chaoyang,2", + "d1001,2018-10-03 14:38:16.800,12.30000,221,0.31000,Beijing.Chaoyang,2", + "d1003,2018-10-03 14:38:16.600,13.40000,223,0.29000,Beijing.Haidian,2"] + + +def get_connection() -> taos.TaosConnection: + """ + create connection use firstEp in taos.cfg and use default user and password. + """ + return taos.connect() + + +def create_stable(conn: taos.TaosConnection): + conn.execute("CREATE DATABASE power") + conn.execute("USE power") + conn.execute("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " + "TAGS (location BINARY(64), groupId INT)") + + +# The generated SQL is: +# INSERT INTO d1001 USING meters TAGS(Beijing.Chaoyang, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) +# d1002 USING meters TAGS(Beijing.Chaoyang, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) +# d1003 USING meters TAGS(Beijing.Haidian, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) +# d1004 USING meters TAGS(Beijing.Haidian, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000) + +def get_sql(): + global lines + lines = map(lambda line: line.split(','), lines) # [['d1001', ...]...] + lines = sorted(lines, key=lambda ls: ls[0]) # sort by table name + sql = "INSERT INTO " + tb_name = None + for ps in lines: + tmp_tb_name = ps[0] + if tb_name != tmp_tb_name: + tb_name = tmp_tb_name + sql += f"{tb_name} USING meters TAGS({ps[5]}, {ps[6]}) VALUES " + sql += f"('{ps[1]}', {ps[2]}, {ps[3]}, {ps[4]}) " + return sql + + +def insert_data(conn: taos.TaosConnection): + sql = get_sql() + affected_rows = conn.execute(sql) + print("affected_rows", affected_rows) # 8 + + +if __name__ == '__main__': + connection = get_connection() + try: + create_stable(connection) + insert_data(connection) + finally: + connection.close() diff --git a/docs-examples/python/query_example.py b/docs-examples/python/query_example.py new file mode 100644 index 0000000000000000000000000000000000000000..6d33c49c968d9210b475931b5d8cecca0ceff3e3 --- /dev/null +++ b/docs-examples/python/query_example.py @@ -0,0 +1,42 @@ +import taos + + +# ANCHOR: iter +def query_api_demo(conn: taos.TaosConnection): + result: taos.TaosResult = conn.query("SELECT tbname, * FROM meters LIMIT 2") + print("field count:", result.field_count) + print("meta of fields[1]:", result.fields[1]) + print("======================Iterate on result=========================") + for row in result: + print(row) + + +# field count: 7 +# meta of files[1]: {name: ts, type: 9, bytes: 8} +# ======================Iterate on result========================= +# ('d1001', datetime.datetime(2018, 10, 3, 14, 38, 5), 10.300000190734863, 219, 0.3100000023841858, 'Beijing.Chaoyang', 2) +# ('d1001', datetime.datetime(2018, 10, 3, 14, 38, 15), 12.600000381469727, 218, 0.33000001311302185, 'Beijing.Chaoyang', 2) +# ANCHOR_END: iter + +# ANCHOR: fetch_all +def fetch_all_demo(conn: taos.TaosConnection): + result: taos.TaosResult = conn.query("SELECT ts, current FROM meters LIMIT 2") + rows = result.fetch_all_into_dict() + print("row count:", result.row_count) + print("===============all data===================") + print(rows) + + +# row count: 2 +# ===============all data=================== +# [{'ts': datetime.datetime(2018, 10, 3, 14, 38, 5), 'current': 10.300000190734863}, +# {'ts': datetime.datetime(2018, 10, 3, 14, 38, 15), 'current': 12.600000381469727}] +# ANCHOR_END: fetch_all + +if __name__ == '__main__': + connection = taos.connect(database="power") + try: + query_api_demo(connection) + fetch_all_demo(connection) + finally: + connection.close() diff --git a/docs-examples/python/rest_client_example.py b/docs-examples/python/rest_client_example.py new file mode 100644 index 0000000000000000000000000000000000000000..46d33a1d795f8c8dbb0b830061d43ed4510046ba --- /dev/null +++ b/docs-examples/python/rest_client_example.py @@ -0,0 +1,9 @@ +from taosrest import RestClient + +client = RestClient("localhost", 6041, "root", "taosdata") +res: dict = client.sql("SELECT ts, current FROM power.meters LIMIT 1") +print(res) + +# output: +# {'status': 'succ', 'head': ['ts', 'current'], 'column_meta': [['ts', 9, 8], ['current', 6, 4]], 'data': [[datetime.datetime(2018, 10, 3, 14, 38, 5, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 10.3]], 'rows': 1} + diff --git a/docs-examples/python/result_set_examples.py b/docs-examples/python/result_set_examples.py new file mode 100644 index 0000000000000000000000000000000000000000..6cba0d3f73cdaa408309de93652ed8f346c8f759 --- /dev/null +++ b/docs-examples/python/result_set_examples.py @@ -0,0 +1,33 @@ +import taos + +conn = taos.connect() +conn.execute("DROP DATABASE IF EXISTS test") +conn.execute("CREATE DATABASE test") +conn.select_db("test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") +# prepare data +for i in range(2000): + location = str(i % 10) + tb = "t" + location + conn.execute(f"INSERT INTO {tb} USING weather TAGS({location}) VALUES (now+{i}a, 23.5) (now+{i + 1}a, 23.5)") + +result: taos.TaosResult = conn.query("SELECT * FROM weather") + +block_index = 0 +blocks: taos.TaosBlocks = result.blocks_iter() +for rows, length in blocks: + print("block ", block_index, " length", length) + print("first row in this block:", rows[0]) + block_index += 1 + +conn.close() + +# possible output: +# block 0 length 1200 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 46000), 23.5, 0) +# block 1 length 1200 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 76000), 23.5, 3) +# block 2 length 1200 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 99000), 23.5, 6) +# block 3 length 400 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 122000), 23.5, 9) diff --git a/docs-examples/python/subscribe_demo.py b/docs-examples/python/subscribe_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..db9d49c3f4f8122634800c02a683d4cb022a7ba0 --- /dev/null +++ b/docs-examples/python/subscribe_demo.py @@ -0,0 +1,38 @@ +""" +Python asynchronous subscribe demo. +run on Linux system with: python3 subscribe_demo.py +""" + +from ctypes import c_void_p + +import taos +import time + + +def query_callback(p_sub, p_result, p_param, code): + """ + :param p_sub: pointer returned by native API -- taos_subscribe + :param p_result: pointer to native TAOS_RES + :param p_param: None + :param code: error code + :return: None + """ + print("in callback") + result = taos.TaosResult(c_void_p(p_result)) + # raise exception if error occur + result.check_error(code) + for row in result.rows_iter(): + print(row) + print(f"{result.row_count} rows consumed.") + + +if __name__ == '__main__': + conn = taos.connect() + restart = True + topic = "topic-meter-current-bg" + sql = "select * from power.meters where current > 10" # Error sql + interval = 2000 # consumption interval in microseconds. + _ = conn.subscribe(restart, topic, sql, interval, query_callback) + # Note: we received the return value as _ above, to avoid the TaosSubscription object to be deleted by gc. + while True: + time.sleep(10) # use Ctrl + C to interrupt diff --git a/docs-examples/python/telnet_line_protocol_example.py b/docs-examples/python/telnet_line_protocol_example.py new file mode 100644 index 0000000000000000000000000000000000000000..072835109ee238940e6fe5880b72b2b04e0157fa --- /dev/null +++ b/docs-examples/python/telnet_line_protocol_example.py @@ -0,0 +1,38 @@ +import taos +from taos import SmlProtocol, SmlPrecision + +# format: =[ =] +lines = ["meters.current 1648432611249 10.3 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611250 12.6 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611249 10.8 location=Beijing.Haidian groupid=3", + "meters.current 1648432611250 11.3 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611249 219 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611250 218 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611249 221 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611250 217 location=Beijing.Haidian groupid=3", + ] + + +# create connection use firstEp in taos.cfg. +def get_connection(): + return taos.connect() + + +def create_database(conn): + conn.execute("CREATE DATABASE test") + conn.execute("USE test") + + +def insert_lines(conn): + affected_rows = conn.schemaless_insert( + lines, SmlProtocol.TELNET_PROTOCOL, SmlPrecision.NOT_CONFIGURED) + print(affected_rows) # 8 + + +if __name__ == '__main__': + connection = get_connection() + try: + create_database(connection) + insert_lines(connection) + finally: + connection.close() diff --git a/docs-examples/rust/Cargo.toml b/docs-examples/rust/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..114407e69edcb94fdeef77e6ae581569c5451160 --- /dev/null +++ b/docs-examples/rust/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = ["restexample", "nativeexample", "schemalessexample"] diff --git a/docs-examples/rust/nativeexample/Cargo.toml b/docs-examples/rust/nativeexample/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..64fd10a3e915a39c321b56b6f38be51417d8d18e --- /dev/null +++ b/docs-examples/rust/nativeexample/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "nativeexample" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +libtaos = { version = "0.4.3" } +tokio = { version = "*", features = ["rt", "macros", "rt-multi-thread"] } +bstr = { version = "*" } diff --git a/docs-examples/rust/nativeexample/examples/connect.rs b/docs-examples/rust/nativeexample/examples/connect.rs new file mode 100644 index 0000000000000000000000000000000000000000..8e27458de5d428479668a0e5133ca904cf27c213 --- /dev/null +++ b/docs-examples/rust/nativeexample/examples/connect.rs @@ -0,0 +1,19 @@ +use libtaos::*; + +fn taos_connect() -> Result { + TaosCfgBuilder::default() + .ip("localhost") + .user("root") + .pass("taosdata") + // .db("log") // remove comment if you want to connect to database log by default. + .port(6030u16) + .build() + .expect("TaosCfg builder error") + .connect() +} + +fn main() { + #[allow(unused_variables)] + let taos = taos_connect().unwrap(); + println!("Connected") +} diff --git a/docs-examples/rust/nativeexample/examples/stmt_example.rs b/docs-examples/rust/nativeexample/examples/stmt_example.rs new file mode 100644 index 0000000000000000000000000000000000000000..a791a4135984a33dded145e8175d7ade57de8d77 --- /dev/null +++ b/docs-examples/rust/nativeexample/examples/stmt_example.rs @@ -0,0 +1,38 @@ +use bstr::BString; +use libtaos::*; + +#[tokio::main] +async fn main() -> Result<(), Error> { + let taos = TaosCfg::default().connect().expect("fail to connect"); + taos.create_database("power").await?; + taos.use_database("power").await?; + taos.exec("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)").await?; + let mut stmt = taos.stmt("INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)")?; + // bind table name and tags + stmt.set_tbname_tags( + "d1001", + [ + Field::Binary(BString::from("Beijing.Chaoyang")), + Field::Int(2), + ], + )?; + // bind values. + let values = vec![ + Field::Timestamp(Timestamp::new(1648432611249, TimestampPrecision::Milli)), + Field::Float(10.3), + Field::Int(219), + Field::Float(0.31), + ]; + stmt.bind(&values)?; + // bind one more row + let values2 = vec![ + Field::Timestamp(Timestamp::new(1648432611749, TimestampPrecision::Milli)), + Field::Float(12.6), + Field::Int(218), + Field::Float(0.33), + ]; + stmt.bind(&values2)?; + // execute + stmt.execute()?; + Ok(()) +} diff --git a/docs-examples/rust/nativeexample/examples/subscribe_demo.rs b/docs-examples/rust/nativeexample/examples/subscribe_demo.rs new file mode 100644 index 0000000000000000000000000000000000000000..4febff9be7bd5771db449fbfb184a9f208e61d8a --- /dev/null +++ b/docs-examples/rust/nativeexample/examples/subscribe_demo.rs @@ -0,0 +1,3 @@ +fn main() { + +} \ No newline at end of file diff --git a/docs-examples/rust/nativeexample/src/main.rs b/docs-examples/rust/nativeexample/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..e7a11a969c037e00a796aafeff6258501ec15e9a --- /dev/null +++ b/docs-examples/rust/nativeexample/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/docs-examples/rust/restexample/Cargo.toml b/docs-examples/rust/restexample/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..a5f89f8a3be3baabd298a70947f6c5d3df088aae --- /dev/null +++ b/docs-examples/rust/restexample/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "restexample" +version = "0.1.0" +edition = "2021" + +[dependencies] +libtaos = { version = "0.4.3", features = ["rest"] } +tokio = { version = "*", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/docs-examples/rust/restexample/examples/connect.rs b/docs-examples/rust/restexample/examples/connect.rs new file mode 100644 index 0000000000000000000000000000000000000000..b3718342c4a142786572a7eec46d6fa36f651566 --- /dev/null +++ b/docs-examples/rust/restexample/examples/connect.rs @@ -0,0 +1,20 @@ +use libtaos::*; + +fn taos_connect() -> Result { + TaosCfgBuilder::default() + .ip("localhost") + .user("root") + .pass("taosdata") + // .db("log") // remove comment if you want to connect to database log by default. + .port(6030u16) + .build() + .expect("TaosCfg builder error") + .connect() +} + +#[tokio::main] +async fn main() { + #[allow(unused_variables)] + let taos = taos_connect().expect("connect error"); + println!("Connected") +} diff --git a/docs-examples/rust/restexample/examples/insert_example.rs b/docs-examples/rust/restexample/examples/insert_example.rs new file mode 100644 index 0000000000000000000000000000000000000000..d7acc98d096fb3cd6bea22d6c5f6f0f5caea50af --- /dev/null +++ b/docs-examples/rust/restexample/examples/insert_example.rs @@ -0,0 +1,18 @@ +use libtaos::*; + +#[tokio::main] +async fn main() -> Result<(), Error> { + let taos = TaosCfg::default().connect().expect("fail to connect"); + taos.create_database("power").await?; + taos.exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)").await?; + let sql = "INSERT INTO power.d1001 USING power.meters TAGS(Beijing.Chaoyang, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS(Beijing.Chaoyang, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS(Beijing.Haidian, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS(Beijing.Haidian, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; + let result = taos.query(sql).await?; + println!("{:?}", result); + Ok(()) +} + +// output: +// TaosQueryData { column_meta: [ColumnMeta { name: "affected_rows", type_: Int, bytes: 4 }], rows: [[Int(8)]] } diff --git a/docs-examples/rust/restexample/examples/query_example.rs b/docs-examples/rust/restexample/examples/query_example.rs new file mode 100644 index 0000000000000000000000000000000000000000..bbe0cfaabf0d3b078606e823dd504be4153832e7 --- /dev/null +++ b/docs-examples/rust/restexample/examples/query_example.rs @@ -0,0 +1,39 @@ +use libtaos::*; + +fn taos_connect() -> Result { + TaosCfgBuilder::default() + .ip("localhost") + .user("root") + .pass("taosdata") + .db("power") + .port(6030u16) + .build() + .expect("TaosCfg builder error") + .connect() +} + +#[tokio::main] +async fn main() -> Result<(), Error> { + let taos = taos_connect().expect("connect error"); + let result = taos.query("SELECT ts, current FROM meters LIMIT 2").await?; + // print column names + let meta: Vec = result.column_meta; + for column in meta { + print!("{}\t", column.name) + } + println!(); + // print rows + let rows: Vec> = result.rows; + for row in rows { + for field in row { + print!("{}\t", field); + } + println!(); + } + Ok(()) +} + +// output: +// ts current +// 2022-03-28 09:56:51.249 10.3 +// 2022-03-28 09:56:51.749 12.6 diff --git a/docs-examples/rust/restexample/src/main.rs b/docs-examples/rust/restexample/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..e7a11a969c037e00a796aafeff6258501ec15e9a --- /dev/null +++ b/docs-examples/rust/restexample/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/docs-examples/rust/schemalessexample/Cargo.toml b/docs-examples/rust/schemalessexample/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..32c6a122318d16f488210da54f1600ba8f6a8b7c --- /dev/null +++ b/docs-examples/rust/schemalessexample/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "schemalessexample" +version = "0.1.0" +edition = "2021" + +[dependencies] +libtaos = { version = "0.4.3" } diff --git a/docs-examples/rust/schemalessexample/examples/influxdb_line_example.rs b/docs-examples/rust/schemalessexample/examples/influxdb_line_example.rs new file mode 100644 index 0000000000000000000000000000000000000000..e93888cc83d12f3bec7370a66e8a85d38cec42ad --- /dev/null +++ b/docs-examples/rust/schemalessexample/examples/influxdb_line_example.rs @@ -0,0 +1,22 @@ +use libtaos::schemaless::*; +use libtaos::*; + +fn main() { + let taos = TaosCfg::default().connect().expect("fail to connect"); + taos.raw_query("CREATE DATABASE test").unwrap(); + taos.raw_query("USE test").unwrap(); + let lines = ["meters,location=Beijing.Haidian,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249", + "meters,location=Beijing.Haidian,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250", + "meters,location=Beijing.Haidian,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249", + "meters,location=Beijing.Haidian,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250"]; + let affected_rows = taos + .schemaless_insert( + &lines, + TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLISECONDS, + ) + .unwrap(); + println!("affected_rows={}", affected_rows); +} + +// run with: cargo run --example influxdb_line_example diff --git a/docs-examples/rust/schemalessexample/examples/opentsdb_json_example.rs b/docs-examples/rust/schemalessexample/examples/opentsdb_json_example.rs new file mode 100644 index 0000000000000000000000000000000000000000..1d66bd1f2b1bcbe82dc3ee3e8e25ea4c521c81f0 --- /dev/null +++ b/docs-examples/rust/schemalessexample/examples/opentsdb_json_example.rs @@ -0,0 +1,25 @@ +use libtaos::schemaless::*; +use libtaos::*; + +fn main() { + let taos = TaosCfg::default().connect().expect("fail to connect"); + taos.raw_query("CREATE DATABASE test").unwrap(); + taos.raw_query("USE test").unwrap(); + let lines = [ + r#"[{"metric": "meters.current", "timestamp": 1648432611249, "value": 10.3, "tags": {"location": "Beijing.Chaoyang", "groupid": 2}}, + {"metric": "meters.voltage", "timestamp": 1648432611249, "value": 219, "tags": {"location": "Beijing.Haidian", "groupid": 1}}, + {"metric": "meters.current", "timestamp": 1648432611250, "value": 12.6, "tags": {"location": "Beijing.Chaoyang", "groupid": 2}}, + {"metric": "meters.voltage", "timestamp": 1648432611250, "value": 221, "tags": {"location": "Beijing.Haidian", "groupid": 1}}]"#, + ]; + + let affected_rows = taos + .schemaless_insert( + &lines, + TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NOT_CONFIGURED, + ) + .unwrap(); + println!("affected_rows={}", affected_rows); // affected_rows=4 +} + +// run with: cargo run --example opentsdb_json_example diff --git a/docs-examples/rust/schemalessexample/examples/opentsdb_telnet_example.rs b/docs-examples/rust/schemalessexample/examples/opentsdb_telnet_example.rs new file mode 100644 index 0000000000000000000000000000000000000000..18d7500714d9e41b1bebd490199d296ead3dc7c4 --- /dev/null +++ b/docs-examples/rust/schemalessexample/examples/opentsdb_telnet_example.rs @@ -0,0 +1,28 @@ +use libtaos::schemaless::*; +use libtaos::*; + +fn main() { + let taos = TaosCfg::default().connect().expect("fail to connect"); + taos.raw_query("CREATE DATABASE test").unwrap(); + taos.raw_query("USE test").unwrap(); + let lines = [ + "meters.current 1648432611249 10.3 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611250 12.6 location=Beijing.Chaoyang groupid=2", + "meters.current 1648432611249 10.8 location=Beijing.Haidian groupid=3", + "meters.current 1648432611250 11.3 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611249 219 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611250 218 location=Beijing.Chaoyang groupid=2", + "meters.voltage 1648432611249 221 location=Beijing.Haidian groupid=3", + "meters.voltage 1648432611250 217 location=Beijing.Haidian groupid=3", + ]; + let affected_rows = taos + .schemaless_insert( + &lines, + TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NOT_CONFIGURED, + ) + .unwrap(); + println!("affected_rows={}", affected_rows); // affected_rows=8 +} + +// run with: cargo run --example opentsdb_telnet_example diff --git a/docs-examples/rust/schemalessexample/src/main.rs b/docs-examples/rust/schemalessexample/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..e7a11a969c037e00a796aafeff6258501ec15e9a --- /dev/null +++ b/docs-examples/rust/schemalessexample/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/example/src/tmq.c b/example/src/tmq.c index 1abce3f188fcefd77b4f0d0037284b55d79892b0..913096ee90294cf65ba81d605ed3e7d4f2fa803c 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -106,8 +106,8 @@ int32_t create_topic() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create topic topic_ctb_column as abc1"); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -167,9 +167,11 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.pass", "taosdata"); /*tmq_conf_set(conf, "td.connect.db", "abc1");*/ tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "false"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); + tmq_conf_destroy(conf); return tmq; } @@ -239,7 +241,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { msg_process(tmqmessage); taos_free_result(tmqmessage); - tmq_commit_async(tmq, NULL, tmq_commit_cb_print, NULL); + /*tmq_commit_async(tmq, NULL, tmq_commit_cb_print, NULL);*/ /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ } } diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 72d2c142d2dafd55fac72ae8f0b0b58aad69b40d..1d32c9825f09fe018895dd54e1dfa8b6b84a257a 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -86,6 +86,11 @@ typedef enum { TSDB_RETENTION_MAX = 3 } ERetentionLevel; +typedef enum { + TSDB_BITMODE_DEFAULT = 0, // 2 bits + TSDB_BITMODE_ONE_BIT = 1, // 1 bit +} EBitmapMode; + extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index cea40f4785be8e32a07fadf165a7407d56fddf8e..db8644ecfed50f354e61ff20b424f93dc559f8d7 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -190,7 +190,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); -int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); +int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); void colDataTrim(SColumnInfoData* pColumnInfoData); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 29e9695e3edd09e15e06ec0559c38750f51f19db..bbe7d22cfb824c54323c7c5801ba05fac9f0d4b1 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -314,8 +314,9 @@ typedef struct { SDataCol *cols; } SDataCols; -static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != 0; } -static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = 1; } +static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != TSDB_BITMODE_DEFAULT; } +static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = TSDB_BITMODE_ONE_BIT; } +static FORCE_INLINE bool tdIsBitmapModeI(int8_t bitmapMode) { return bitmapMode != TSDB_BITMODE_DEFAULT; } #define keyCol(pCols) (&((pCols)->cols[0])) // Key column #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] // the idx row of column-wised data diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 111241cf03a9da14f4f62c47206b8a990a6a529b..2a4ef565dd1c6b6742446adee2daf953665b99e5 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -128,6 +128,7 @@ extern bool tsStartUdfd; // schemaless extern char tsSmlChildTableName[]; +extern char tsSmlTagName[]; extern bool tsSmlDataFormat; // internal diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9104e8a4234a48692f73d9d4c3225e9cd0668ee0..b89309906417a2876fba667fcaaf15be7219d398 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1352,7 +1352,7 @@ typedef struct { typedef struct { int32_t code; - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t sversion; int32_t tversion; } SResReadyRsp; @@ -1987,19 +1987,16 @@ static FORCE_INLINE void tFreeClientHbReq(void* pReq) { if (req->info) { tFreeReqKvHash(req->info); taosHashCleanup(req->info); + req->info = NULL; } } int32_t tSerializeSClientHbBatchReq(void* buf, int32_t bufLen, const SClientHbBatchReq* pReq); int32_t tDeserializeSClientHbBatchReq(void* buf, int32_t bufLen, SClientHbBatchReq* pReq); -static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { +static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) { SClientHbBatchReq* req = (SClientHbBatchReq*)pReq; - if (deep) { - taosArrayDestroyEx(req->reqs, tFreeClientHbReq); - } else { - taosArrayDestroy(req->reqs); - } + taosArrayDestroyEx(req->reqs, tFreeClientHbReq); taosMemoryFree(pReq); } @@ -2023,6 +2020,7 @@ static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) { int32_t tSerializeSClientHbBatchRsp(void* buf, int32_t bufLen, const SClientHbBatchRsp* pBatchRsp); int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchRsp* pBatchRsp); +void tFreeSClientHbBatchRsp(SClientHbBatchRsp *pBatchRsp); static FORCE_INLINE int32_t tEncodeSKv(SEncoder* pEncoder, const SKv* pKv) { if (tEncodeI32(pEncoder, pKv->key) < 0) return -1; @@ -2523,11 +2521,9 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); buf = taosDecodeFixedI32(buf, &pRsp->blockNum); - pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); if (pRsp->blockNum != 0) { + pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); buf = taosDecodeFixedI8(buf, &pRsp->withTbName); buf = taosDecodeFixedI8(buf, &pRsp->withSchema); buf = taosDecodeFixedI8(buf, &pRsp->withTag); @@ -2540,14 +2536,20 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p taosArrayPush(pRsp->blockDataLen, &bLen); taosArrayPush(pRsp->blockData, &data); if (pRsp->withSchema) { + pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); buf = taosDecodeSSchemaWrapper(buf, pSW); taosArrayPush(pRsp->blockSchema, &pSW); + } else { + pRsp->blockSchema = NULL; } if (pRsp->withTbName) { + pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); char* name = NULL; buf = taosDecodeString(buf, &name); taosArrayPush(pRsp->blockTbName, &name); + } else { + pRsp->blockTbName = NULL; } } } diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 89553f978beaacd7f54cdce5ab786d510ecd9fc7..1ab101f705ac3f71fad134c200a22f903e4a8e86 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -72,7 +72,6 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad); * @param pMsg The request message */ int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg); -int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 389fe25780f5bbdde734ba1248dd5e04c4a14d3f..68a1e08f518f5c5e230076cd56344ea1161804cb 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -198,6 +198,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_HANDLE_ERROR(_code) \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) +#define NEED_CLIENT_RM_TBLMETA_REQ(_type) ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB \ + || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB) #define NEED_SCHEDULER_RETRY_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) @@ -228,23 +230,23 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ } \ } while (0) -#define qDebug(...) \ - do { \ - if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ - } \ +#define qDebug(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qTrace(...) \ - do { \ - if (qDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ - } \ +#define qTrace(...) \ + do { \ + if (qDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qDebugL(...) \ - do { \ - if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ - } \ +#define qDebugL(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + } \ } while (0) #define QRY_ERR_RET(c) \ diff --git a/include/libs/transport/rpcHead.h b/include/libs/transport/rpcHead.h deleted file mode 100644 index 5ddf1a83c96271abba9aefc254b9cee442c375cc..0000000000000000000000000000000000000000 --- a/include/libs/transport/rpcHead.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_RPCHEAD_H -#define TDENGINE_RPCHEAD_H - -#include -#ifdef __cplusplus -extern "C" { -#endif - -#define RPC_CONN_TCP 2 - -extern int tsRpcOverhead; - -typedef struct { - void* msg; - int msgLen; - uint32_t ip; - uint16_t port; - int connType; - void* shandle; - void* thandle; - void* chandle; -} SRecvInfo; - -#pragma pack(push, 1) - -typedef struct { - char version : 4; // RPC version - char comp : 4; // compression algorithm, 0:no compression 1:lz4 - char resflag : 2; // reserved bits - char spi : 3; // security parameter index - char encrypt : 3; // encrypt algorithm, 0: no encryption - uint16_t tranId; // transcation ID - uint32_t linkUid; // for unique connection ID assigned by client - uint64_t ahandle; // ahandle assigned by client - uint32_t sourceId; // source ID, an index for connection list - uint32_t destId; // destination ID, an index for connection list - uint32_t destIp; // destination IP address, for NAT scenario - char user[TSDB_UNI_LEN]; // user ID - uint16_t port; // for UDP only, port may be changed - char empty[1]; // reserved - uint16_t msgType; // message type - int32_t msgLen; // message length including the header iteslf - uint32_t msgVer; - int32_t code; // code in response message - uint8_t content[0]; // message body starts from here -} SRpcHead; - -typedef struct { - int32_t reserved; - int32_t contLen; -} SRpcComp; - -typedef struct { - uint32_t timeStamp; - uint8_t auth[TSDB_AUTH_LEN]; -} SRpcDigest; - -#pragma pack(pop) - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_RPCHEAD_H diff --git a/include/libs/transport/transport.h b/include/libs/transport/transport.h index f5ffc125eaccfcd6431d29cc379b04ed88657d4d..136e8b35cde07e299cceb0d9268bd066cf40b2a3 100644 --- a/include/libs/transport/transport.h +++ b/include/libs/transport/transport.h @@ -20,9 +20,8 @@ extern "C" { #endif - #ifdef __cplusplus } #endif -#endif /*_TD_TRANSPORT_H_*/ \ No newline at end of file +#endif /*_TD_TRANSPORT_H_*/ diff --git a/include/os/osDir.h b/include/os/osDir.h index b549acde3706bd68e4b3f18aa24fb056eb96a189..a4c686e2807ee3d1fb9a8a0e1e05066d1b616c0b 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -31,6 +31,12 @@ extern "C" { #endif +#ifdef WINDOWS +#define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\" +#else +#define TD_TMP_DIR_PATH "/tmp/" +#endif + typedef struct TdDir *TdDirPtr; typedef struct TdDirEntry *TdDirEntryPtr; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 66287099cdc81783c1a7b1ba6e42c8265945cf63..9868c2cc0d1fc9f13fe440438b040c257d089806 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -39,64 +39,50 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SUCCESS 0 #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error -// rpc -#define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) -#define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) -#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) -#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) -#define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) -#define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) -#define TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x0007) -#define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) -#define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) -#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) -#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) -#define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) -#define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) -#define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) -#define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) -#define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) -#define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) -#define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) -#define TSDB_CODE_RPC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0013) -#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) -#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) -#define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) -#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) - //common & util -#define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0100) -#define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0101) -#define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0102) -#define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0103) -#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0104) -#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0105) -#define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0106) -#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0107) -#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0108) -#define TSDB_CODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0109) -#define TSDB_CODE_INVALID_JSON_FORMAT TAOS_DEF_ERROR_CODE(0, 0x010A) -#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x010B) -#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x010C) -#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x010D) -#define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x010E) -#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x010F) -#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0110) -#define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0111) -#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0112) -#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0113) -#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0114) -#define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0115) -#define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0116) -#define TSDB_CODE_NEED_RETRY TAOS_DEF_ERROR_CODE(0, 0x0117) -#define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE TAOS_DEF_ERROR_CODE(0, 0x0118) - -#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0140) -#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0141) -#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0152) -#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0153) -#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0154) -#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0155) +#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) +#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0002) +#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0003) +#define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0010) +#define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0011) +#define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0012) +#define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0013) +#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0014) +#define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0015) +#define TSDB_CODE_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0016) +#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0017) +#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0018) +#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0019) +#define TSDB_CODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x001A) +#define TSDB_CODE_INVALID_JSON_FORMAT TAOS_DEF_ERROR_CODE(0, 0x001B) +#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x001C) +#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x001D) +#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x001E) +#define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x001F) +#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0020) +#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0021) +#define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0022) +#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0023) +#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0024) +#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0025) +#define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0026) +#define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0027) +#define TSDB_CODE_NEED_RETRY TAOS_DEF_ERROR_CODE(0, 0x0028) +#define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE TAOS_DEF_ERROR_CODE(0, 0x0029) + +#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0040) +#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0041) +#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0042) +#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0043) +#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0044) +#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0045) + +// rpc +#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0100) +#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0101) +#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x0102) +#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0103) +#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0104) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) @@ -143,9 +129,8 @@ int32_t* taosGetErrno(); // mnode-common #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) #define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0301) -#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0302) -#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) -#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0304) +#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0302) +#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0303) // mnode-show #define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310) @@ -296,8 +281,8 @@ int32_t* taosGetErrno(); // dnode #define TSDB_CODE_NODE_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_NODE_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0401) -#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0403) -#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0404) +#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0402) +#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0403) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) @@ -434,6 +419,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) #define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) #define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) +#define TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0A0B) // wal #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) @@ -651,6 +637,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_COMMENT_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x264E) #define TSDB_CODE_PAR_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x264F) #define TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY TAOS_DEF_ERROR_CODE(0, 0x2650) +#define TSDB_CODE_PAR_INVALID_DROP_COL TAOS_DEF_ERROR_CODE(0, 0x2651) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index c4dc98354ec5543e666bb719a1d4f580c015ccdc..669b2bc97eb3e6fab04701aebbf80402432b44c1 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -60,7 +60,7 @@ static void registerRequest(SRequestObj *pRequest) { static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj * pTscObj = pRequest->pTscObj; + STscObj *pTscObj = pRequest->pTscObj; SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); @@ -313,7 +313,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return 0; } - SConfig * pCfg = taosGetCfg(); + SConfig *pCfg = taosGetCfg(); SConfigItem *pItem = NULL; switch (option) { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 13ac43bc392d14407bfc04b494f4e5d49a3f1fa3..d01ec501ba215dae820a72a8dfa8ab473d5b8950 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -310,6 +310,8 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { taosArrayDestroy(desc.subDesc); desc.subDesc = NULL; } + } else { + desc.subDesc = NULL; } releaseRequest(*rid); @@ -394,6 +396,10 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S tscDebug("hb got %d expired users, valueLen:%d", userNum, kv.valueLen); + if (NULL == req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + } + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); return TSDB_CODE_SUCCESS; @@ -429,6 +435,10 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen); + if (NULL == req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + } + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); return TSDB_CODE_SUCCESS; @@ -463,6 +473,10 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen); + if (NULL == req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + } + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); return TSDB_CODE_SUCCESS; @@ -511,16 +525,6 @@ static FORCE_INLINE void hbMgrInitHandle() { hbMgrInitMqHbHandle(); } -void hbFreeReq(void *req) { - SClientHbReq *pReq = (SClientHbReq *)req; - tFreeReqKvHash(pReq->info); -} - -void hbClearClientHbReq(SClientHbReq *pReq) { - pReq->query = NULL; - pReq->info = NULL; -} - SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { @@ -535,6 +539,8 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { while (pIter != NULL) { SClientHbReq *pOneReq = pIter; + pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); + SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pOneReq->connKey, sizeof(SClientHbKey)); if (info) { code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, info->param, pOneReq); @@ -544,7 +550,6 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { } } - taosArrayPush(pBatchReq->reqs, pOneReq); //hbClearClientHbReq(pOneReq); pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); @@ -601,8 +606,8 @@ static void *hbThreadFunc(void *param) { void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tFreeClientHbBatchReq(pReq, false); - hbClearReqInfo(pAppHbMgr); + tFreeClientHbBatchReq(pReq); + //hbClearReqInfo(pAppHbMgr); break; } @@ -611,8 +616,8 @@ static void *hbThreadFunc(void *param) { if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tFreeClientHbBatchReq(pReq, false); - hbClearReqInfo(pAppHbMgr); + tFreeClientHbBatchReq(pReq); + //hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); break; } @@ -628,8 +633,8 @@ static void *hbThreadFunc(void *param) { int64_t transporterId = 0; SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); - tFreeClientHbBatchReq(pReq, false); - hbClearReqInfo(pAppHbMgr); + tFreeClientHbBatchReq(pReq); + //hbClearReqInfo(pAppHbMgr); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } @@ -721,8 +726,7 @@ void appHbMgrCleanup(void) { void *pIter = taosHashIterate(pTarget->activeInfo, NULL); while (pIter != NULL) { SClientHbReq *pOneReq = pIter; - hbFreeReq(pOneReq); - taosHashCleanup(pOneReq->info); + tFreeClientHbReq(pOneReq); pIter = taosHashIterate(pTarget->activeInfo, pIter); } taosHashCleanup(pTarget->activeInfo); @@ -782,7 +786,7 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo * } SClientHbReq hbReq = {0}; hbReq.connKey = connKey; - hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + //hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq)); @@ -823,8 +827,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (pReq) { - hbFreeReq(pReq); - taosHashCleanup(pReq->info); + tFreeClientHbReq(pReq); taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 386283b5b58d04b68a00f89100d64b1312e82b46..b1428858418a499a0260376eb4b1b6af60c06ea7 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -345,6 +345,10 @@ int32_t validateSversion(SRequestObj* pRequest, void* res) { for (int32_t i = 0; i < pRsp->nBlocks; ++i) { SSubmitBlkRsp* blk = pRsp->pBlocks + i; + if (NULL == blk->tblFName || 0 == blk->tblFName[0]) { + continue; + } + STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; taosArrayPush(pArray, &tbSver); } @@ -383,14 +387,14 @@ _return: } void freeRequestRes(SRequestObj* pRequest, void* res) { - if (NULL == res) { + if (NULL == pRequest || NULL == res) { return; } if (TDMT_VND_SUBMIT == pRequest->type) { tFreeSSubmitRsp((SSubmitRsp*)res); } else if (TDMT_VND_QUERY == pRequest->type) { - taosArrayDestroy((SArray *)res); + taosArrayDestroy((SArray*)res); } } @@ -431,12 +435,13 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; - freeRequestRes(pRequest, pRes); - pRes = NULL; } if (res) { *res = pRes; + } else { + freeRequestRes(pRequest, pRes); + pRes = NULL; } return pRequest; @@ -499,6 +504,23 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { return code; } +int32_t removeMeta(STscObj* pTscObj, SArray* tbList) { + SCatalog* pCatalog = NULL; + int32_t tbNum = taosArrayGetSize(tbList); + int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + for (int32_t i = 0; i < tbNum; ++i) { + SName* pTbName = taosArrayGet(tbList, i); + catalogRemoveTableMeta(pCatalog, pTbName); + } + + return TSDB_CODE_SUCCESS; +} + + SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { SRequestObj* pRequest = NULL; int32_t retryNum = 0; @@ -518,6 +540,10 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { } } while (retryNum++ < REQUEST_MAX_TRY_TIMES); + if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { + removeMeta(pTscObj, pRequest->tableList); + } + return pRequest; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 7360b054e2516c696a316156f73f3474346fa7b2..eec8cab7ad65569f67c63ad5963333550c794ef9 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -146,10 +146,10 @@ void taos_free_result(TAOS_RES *res) { SMqRspObj *pRsp = (SMqRspObj *)res; if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen); - if (pRsp->rsp.blockSchema) taosArrayDestroy(pRsp->rsp.blockSchema); - if (pRsp->rsp.blockTbName) taosArrayDestroy(pRsp->rsp.blockTbName); if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags); if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema); + if (pRsp->rsp.withTbName) taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); + if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); pRsp->resInfo.pRspMsg = NULL; doFreeReqResultInfo(&pRsp->resInfo); } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index e884748fff3fc67963059bb5d1dbcc969318a874..68c47c2d13421cd34e327db37e31ae76774985ac 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -63,10 +63,6 @@ for (int i = 1; i < keyLen; ++i) { \ #define TS "_ts" #define TS_LEN 3 -#define TAG "_tag" -#define TAG_LEN 4 -#define TAG_VALUE "NULL" -#define TAG_VALUE_LEN 4 #define VALUE "value" #define VALUE_LEN 5 @@ -263,7 +259,7 @@ static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSi memcpy(tname, field->key, field->keyLen); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { int32_t bytes = field->length > CHAR_SAVE_LENGTH ? (2*field->length) : CHAR_SAVE_LENGTH; - int out = snprintf(buf, bufSize,"`%s` %s(%d)", + int out = snprintf(buf, bufSize, "`%s` %s(%d)", tname, tDataTypes[field->type].name, bytes); *outBytes = out; } else { @@ -400,6 +396,12 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; } + if(taosArrayGetSize(cols) == 0){ + outBytes = snprintf(pos, freeBytes,"`%s` %s(%d)", + tsSmlTagName, tDataTypes[TSDB_DATA_TYPE_NCHAR].name, CHAR_SAVE_LENGTH); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } pos--; ++freeBytes; outBytes = snprintf(pos, freeBytes, ")"); TAOS_RES* res = taos_query(info->taos, result); @@ -724,9 +726,6 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { if(value + len != endPtr){ return -1; } - if(tsInt64 == 0){ - return taosGetTimestampNs(); - } double ts = tsInt64; switch (type) { case TSDB_TIME_PRECISION_HOURS: @@ -792,8 +791,8 @@ static int8_t smlGetTsTypeByPrecision(int8_t precision) { } static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t len){ - if(len == 0){ - return taosGetTimestamp(TSDB_TIME_PRECISION_NANO); + if(len == 0 || (len == 1 && data[0] == '0')){ + return taosGetTimestampNs(); } int8_t tsType = smlGetTsTypeByPrecision(info->precision); @@ -815,6 +814,9 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle* info, const char* data, int32_t smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); return -1; } + if(len == 1 && data[0] == '0'){ + return taosGetTimestampNs(); + } int8_t tsType = smlGetTsTypeByLen(len); if (tsType == -1) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); @@ -1112,14 +1114,6 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, char *childTableName, bool isTag, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ if(isTag && len == 0){ - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if(!kv) return TSDB_CODE_OUT_OF_MEMORY; - kv->key = TAG; - kv->keyLen = TAG_LEN; - kv->value = TAG_VALUE; - kv->length = TAG_VALUE_LEN; - kv->type = TSDB_DATA_TYPE_NCHAR; - if(cols) taosArrayPush(cols, &kv); return TSDB_CODE_SUCCESS; } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 764571e71e5f3ddf89f58183a1d7b26a73cfc79d..17468584822361d8a3f7ff048cb797a8174b1836 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -47,8 +47,14 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { } break; case STMT_EXECUTE: - if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) { - code = TSDB_CODE_TSC_STMT_API_ERROR; + if (STMT_TYPE_QUERY == pStmt->sql.type) { + if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS) && STMT_STATUS_NE(BIND) && STMT_STATUS_NE(BIND_COL)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } + } else { + if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } } break; default: @@ -794,6 +800,7 @@ int stmtExec(TAOS_STMT* stmt) { if (code) { pStmt->exec.pRequest->code = code; } else { + tFreeSSubmitRsp(pRsp); STMT_ERR_RET(stmtResetStmt(pStmt)); STMT_ERR_RET(TSDB_CODE_NEED_RETRY); } @@ -811,11 +818,13 @@ _return: if (TSDB_CODE_SUCCESS == code && autoCreateTbl) { if (NULL == pRsp) { tscError("no submit resp got for auto create table"); - STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + code = TSDB_CODE_TSC_APP_ERROR; + } else { + code = stmtUpdateTableUid(pStmt, pRsp); } - - STMT_ERR_RET(stmtUpdateTableUid(pStmt, pRsp)); } + + tFreeSSubmitRsp(pRsp); ++pStmt->sql.runTimes; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 36c0d8156c5fd82b55517e87d2fab06a80e2a4a3..dfa56f80c457783eb58255f3a0d494936b475bad 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -202,7 +202,12 @@ tmq_conf_t* tmq_conf_new() { } void tmq_conf_destroy(tmq_conf_t* conf) { - if (conf) taosMemoryFree(conf); + if (conf) { + if (conf->ip) taosMemoryFree(conf->ip); + if (conf->user) taosMemoryFree(conf->user); + if (conf->pass) taosMemoryFree(conf->pass); + taosMemoryFree(conf); + } } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { @@ -497,6 +502,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { } else { ASSERT(0); } + taosFreeQitem(pTaskType); } taosFreeQall(qall); return 0; @@ -954,8 +960,12 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqClientVg* pVg = pParam->pVg; SMqClientTopic* pTopic = pParam->pTopic; tmq_t* tmq = pParam->tmq; + int32_t vgId = pParam->vgId; + int32_t epoch = pParam->epoch; + taosMemoryFree(pParam); if (code != 0) { - tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code); + tscWarn("msg discard from vg %d, epoch %d, code:%x", vgId, epoch, code); + if (pMsg->pData) taosMemoryFree(pMsg->pData); goto CREATE_MSG_FAIL; } @@ -963,19 +973,21 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { // do not write into queue since updating epoch reset - tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch, + tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch); tsem_post(&tmq->rspSem); + taosMemoryFree(pMsg->pData); return 0; } if (msgEpoch != tmqEpoch) { - tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); + tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch); } SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { - tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch); + taosMemoryFree(pMsg->pData); + tscWarn("msg discard from vg %d, epoch %d since out of memory", vgId, epoch); goto CREATE_MSG_FAIL; } @@ -986,6 +998,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg); + taosMemoryFree(pMsg->pData); tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId, pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset); @@ -995,7 +1008,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { return 0; CREATE_MSG_FAIL: - if (pParam->epoch == tmq->epoch) { + if (epoch == tmq->epoch) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } tsem_post(&tmq->rspSem); @@ -1088,6 +1101,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; + int8_t async = pParam->async; pParam->code = code; if (code != 0) { tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async); @@ -1104,7 +1118,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { goto END; } - if (!pParam->async) { + if (!async) { SMqAskEpRsp rsp; tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ @@ -1125,13 +1139,14 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { taosWriteQitem(tmq->mqueue, pWrapper); tsem_post(&tmq->rspSem); - taosMemoryFree(pParam); } END: /*atomic_store_8(&tmq->epStatus, 0);*/ - if (!pParam->async) { + if (!async) { tsem_post(&pParam->rspSem); + } else { + taosMemoryFree(pParam); } return code; } @@ -1279,7 +1294,6 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); } - taosFreeQitem(pWrapper); return pRspObj; } @@ -1401,6 +1415,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t waitTime, bool pollIfReset) { } // build rsp SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper); + taosFreeQitem(pollRspWrapper); return pRsp; } else { /*printf("epoch mismatch\n");*/ diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index c7935b351c30912616c6010a46a0fb3bf71130e8..217699e36071e1e4c5e93e391e77a95c4f857af8 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -269,16 +269,7 @@ TEST(testCase, smlParseCols_tag_Test) { ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); size = taosArrayGetSize(cols); - ASSERT_EQ(size, 1); - - // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 0); - ASSERT_EQ(strncasecmp(kv->key, TAG, TAG_LEN), 0); - ASSERT_EQ(kv->keyLen, TAG_LEN); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, TAG_LEN); - ASSERT_EQ(strncasecmp(kv->value, TAG_VALUE, TAG_VALUE_LEN), 0); - taosMemoryFree(kv); + ASSERT_EQ(size, 0); taosArrayDestroy(cols); taosHashCleanup(dumplicateKey); @@ -1207,7 +1198,8 @@ TEST(testCase, sml_TD15662_Test) { ASSERT_NE(info, nullptr); const char *sql[] = { - "hetrey,id=sub_table_0123456,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64", + "hetrey c0=f,c1=127i8 1626006833639", + "hetrey,t1=r c0=f,c1=127i8 1626006833640", }; int ret = smlProcess(info, (char **)sql, sizeof(sql) / sizeof(sql[0])); ASSERT_EQ(ret, 0); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 736cb9854958d3ae8cc0e34321347ef8cb8d0d9d..6e1a9c5726b65291390007515e2cc7b8c1d86dd1 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -341,7 +341,7 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } -int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { +int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) { if (pDataBlock == NULL || pDataBlock->info.rows <= 0) { return 0; } @@ -350,7 +350,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { return -1; } - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0); + int32_t index = (tsColumnIndex == -1)? 0:tsColumnIndex; + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index); if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) { return 0; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f4b8f00e3baf29c0aa72f8fedfb89c12177c7d7f..65c044a29b2558323ad7f0f15ddbba2f339a861b 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -895,7 +895,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { pCols->maxCols = maxCols; pCols->numOfRows = 0; pCols->numOfCols = 0; - // pCols->bitmapMode = 0; // calloc already set 0 + pCols->bitmapMode = TSDB_BITMODE_DEFAULT; if (maxCols > 0) { pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol)); @@ -939,7 +939,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { #endif pCols->numOfRows = 0; - pCols->bitmapMode = 0; + pCols->bitmapMode = TSDB_BITMODE_DEFAULT; pCols->numOfCols = schemaNCols(pSchema); for (i = 0; i < schemaNCols(pSchema); ++i) { @@ -1117,7 +1117,7 @@ void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { int tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size; - if (tlen == 0) return NULL; + // if (tlen == 0) return NULL; // nCols == 0 means no tags tlen += TD_KV_ROW_HEAD_SIZE; @@ -1127,8 +1127,10 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { kvRowSetNCols(row, pBuilder->nCols); kvRowSetLen(row, tlen); - memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols); - memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); + if(pBuilder->nCols > 0){ + memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols); + memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); + } return row; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d74d5a4d4e731cdf1cec4d3e61b2d30a33b467c0..1b61a0bc606aa9fd479cf996668756d2b88f4702 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -78,6 +78,7 @@ char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com"; uint16_t tsTelemPort = 80; // schemaless +char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; //user defined child table name can be specified in tag value. //If set to empty system will generate table name using MD5 hash. bool tsSmlDataFormat = true; // true means that the name and order of cols in each line are the same(only for influx protocol) @@ -326,6 +327,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; + if (cfgAddString(pCfg, "smlTagNullName", tsSmlTagName, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 4; @@ -522,6 +524,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { } tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); + tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagNullName")->str, TSDB_COL_NAME_LEN); tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 0764ea84b9f54b0485e90a2c0bef5b3b5e771a1e..104dee261c9f64c7c8859228dcb0595f4b4df2c0 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -308,13 +308,10 @@ static int compareKv(const void* p1, const void* p2) { * use stable name and tags to grearate child table name */ void buildChildTableName(RandTableName* rName) { - int32_t size = taosArrayGetSize(rName->tags); - ASSERT(size > 0); - taosArraySort(rName->tags, compareKv); - SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen); - for (int j = 0; j < size; ++j) { + taosArraySort(rName->tags, compareKv); + for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { SSmlKv* tagKv = taosArrayGetP(rName->tags, j); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); if(IS_VAR_DATA_TYPE(tagKv->type)){ diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 22bdd960eac7b2e6bcfb6ded211effbdc692f64d..ae41e5d234d0f479772ca78d613c3b80daff460c 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -341,18 +341,19 @@ int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8 bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode) { int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I); uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM]; + uint8_t *qBitmap = (uint8_t*)pBitmap; for (int i = 0; i < nBytes; ++i) { - if (*((uint8_t *)pBitmap) != vTypeByte) { + if (*qBitmap != vTypeByte) { return false; } - pBitmap = POINTER_SHIFT(pBitmap, i); + qBitmap = (uint8_t *)POINTER_SHIFT(pBitmap, i); } int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I); for (int j = 0; j < nLeft; ++j) { uint8_t vType; - tdGetBitmapValType(pBitmap, j, &vType, bitmapMode); + tdGetBitmapValType(qBitmap, j, &vType, bitmapMode); if (vType != TD_VTYPE_NORM) { return false; } diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c index e01992426862492623d2d977120fc5b40e21ed4e..08c9edd854d6a2fa5b1e8ffe98078c86d6436777 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c @@ -47,10 +47,8 @@ static inline void bmSendRsp(SRpcMsg *pMsg, int32_t code) { static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SBnodeMgmt *pMgmt = pInfo->ahandle; - + int32_t code = -1; dTrace("msg:%p, get from bnode-monitor queue", pMsg); - SRpcMsg *pRpc = pMsg; - int32_t code = -1; if (pMsg->msgType == TDMT_MON_BM_INFO) { code = bmProcessGetMonBmInfoReq(pMgmt, pMsg); @@ -58,13 +56,13 @@ static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; } - if (pRpc->msgType & 1U) { + if (IsReq(pMsg)) { if (code != 0 && terrno != 0) code = terrno; bmSendRsp(pMsg, code); } dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pRpc->pCont); + rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 6a7e0ad322efca98991c362d90d602c3ca692c26..bd78fe46effa60c8286cbd901b299f2506de6533 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -19,7 +19,6 @@ static void *dmStatusThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); - setThreadName("dnode-status"); while (1) { @@ -40,7 +39,6 @@ static void *dmStatusThreadFp(void *param) { static void *dmMonitorThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); - setThreadName("dnode-monitor"); while (1) { @@ -103,11 +101,9 @@ void dmStopMonitorThread(SDnodeMgmt *pMgmt) { static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SDnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; - tmsg_t msgType = pMsg->msgType; - bool isRequest = msgType & 1u; - dTrace("msg:%p, will be processed in dnode-mgmt queue, type:%s", pMsg, TMSG_INFO(msgType)); + dTrace("msg:%p, will be processed in dnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - switch (msgType) { + switch (pMsg->msgType) { case TDMT_DND_CONFIG_DNODE: code = dmProcessConfigReq(pMgmt, pMsg); break; @@ -149,7 +145,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { break; } - if (isRequest) { + if (IsReq(pMsg)) { if (code != 0 && terrno != 0) code = terrno; SRpcMsg rsp = { .code = code, diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index c4314a57b15ebc18df872261296911cc62ed07bc..43ee7cd80a5aa1880092a4ca3abe4add2d42249c 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -46,7 +46,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { code = mndProcessMsg(pMsg); } - if (IsReq(pMsg) && pMsg->info.handle != NULL && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (IsReq(pMsg) && pMsg->info.handle != NULL && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && terrno != 0) code = terrno; mmSendRsp(pMsg, code); } @@ -56,28 +56,6 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static void mmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SMnodeMgmt *pMgmt = pInfo->ahandle; - int32_t code = -1; - tmsg_t msgType = pMsg->msgType; - bool isRequest = msgType & 1U; - dTrace("msg:%p, get from mnode-query queue", pMsg); - - pMsg->info.node = pMgmt->pMnode; - code = mndProcessMsg(pMsg); - - if (isRequest) { - if (pMsg->info.handle != NULL && code != 0) { - if (code != 0 && terrno != 0) code = terrno; - mmSendRsp(pMsg, code); - } - } - - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); @@ -135,7 +113,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = tsNumOfMnodeQueryThreads, .max = tsNumOfMnodeQueryThreads, .name = "mnode-query", - .fp = (FItem)mmProcessQueryQueue, + .fp = (FItem)mmProcessQueue, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) { diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 444c42717afdbfa8559fdbd083fc5720f7d4d682..35c94b7fbe786434cfb59191c8899949099d0325 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -19,70 +19,39 @@ static inline void qmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } -static void qmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void qmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SQnodeMgmt *pMgmt = pInfo->ahandle; + int32_t code = -1; + dTrace("msg:%p, get from qnode queue", pMsg); - dTrace("msg:%p, get from qnode-monitor queue", pMsg); - SRpcMsg *pRpc = pMsg; - int32_t code = -1; - - if (pMsg->msgType == TDMT_MON_QM_INFO) { - code = qmProcessGetMonitorInfoReq(pMgmt, pMsg); - } else { - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + switch (pMsg->msgType) { + case TDMT_MON_QM_INFO: + code = qmProcessGetMonitorInfoReq(pMgmt, pMsg); + break; + default: + code = qndProcessQueryMsg(pMgmt->pQnode, pMsg); + break; } - if (pRpc->msgType & 1U) { + if (IsReq(pMsg) && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && terrno != 0) code = terrno; qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pRpc->pCont); - taosFreeQitem(pMsg); -} - -static void qmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SQnodeMgmt *pMgmt = pInfo->ahandle; - - dTrace("msg:%p, get from qnode-query queue", pMsg); - SRpcMsg *pRpc = pMsg; - int32_t code = qndProcessQueryMsg(pMgmt->pQnode, pRpc); - - if (pRpc->msgType & 1U && code != 0) { - qmSendRsp(pMsg, code); - } - - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void qmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SQnodeMgmt *pMgmt = pInfo->ahandle; - - dTrace("msg:%p, get from qnode-fetch queue", pMsg); - SRpcMsg *pRpc = pMsg; - int32_t code = qndProcessFetchMsg(pMgmt->pQnode, pRpc); - - if (pRpc->msgType & 1U && code != 0) { - qmSendRsp(pMsg, code); - } - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } static int32_t qmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { - dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -101,9 +70,7 @@ int32_t qmPutNodeMsgToMonitorQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - if (pMsg == NULL) { - return -1; - } + if (pMsg == NULL) return -1; dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); memcpy(pMsg, pRpc, sizeof(SRpcMsg)); @@ -141,7 +108,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { .min = tsNumOfVnodeQueryThreads, .max = tsNumOfVnodeQueryThreads, .name = "qnode-query", - .fp = (FItem)qmProcessQueryQueue, + .fp = (FItem)qmProcessQueue, .param = pMgmt, }; @@ -154,7 +121,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { .min = tsNumOfQnodeFetchThreads, .max = tsNumOfQnodeFetchThreads, .name = "qnode-fetch", - .fp = (FItem)qmProcessFetchQueue, + .fp = (FItem)qmProcessQueue, .param = pMgmt, }; @@ -167,7 +134,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "qnode-monitor", - .fp = (FItem)qmProcessMonitorQueue, + .fp = (FItem)qmProcessQueue, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index fcfc4f4cee561e00bf4cc31a706898ee75fa2cc4..34a205232e69f057d879188fe4ed98df5b378e3a 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -28,10 +28,8 @@ static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) { static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SSnodeMgmt *pMgmt = pInfo->ahandle; - + int32_t code = -1; dTrace("msg:%p, get from snode-monitor queue", pMsg); - SRpcMsg *pRpc = pMsg; - int32_t code = -1; if (pMsg->msgType == TDMT_MON_SM_INFO) { code = smProcessGetMonitorInfoReq(pMgmt, pMsg); @@ -39,13 +37,13 @@ static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; } - if (pRpc->msgType & 1U) { + if (IsReq(pMsg)) { if (code != 0 && terrno != 0) code = terrno; smSendRsp(pMsg, code); } dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pRpc->pCont); + rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 3a5e8a671ca3e88a10f64cdb1686c3b702eb8d4b..e7bfbdae58bf0556c8e873511476b3b17bd75a73 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -29,7 +29,7 @@ static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) { tmsgSendRsp(&rsp); } -static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void vmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; dTrace("msg:%p, get from vnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); @@ -92,7 +92,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SArray * pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); if (pArray == NULL) { dError("failed to process %d msgs in write-queue since %s", numOfMsgs, terrstr()); return; @@ -222,8 +222,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { - SRpcMsg * pRpc = pMsg; - SMsgHead *pHead = pRpc->pCont; + SMsgHead *pHead = pMsg->pCont; int32_t code = 0; pHead->contLen = ntohl(pHead->contLen); @@ -237,23 +236,23 @@ static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType switch (qtype) { case QUERY_QUEUE: - dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pVnode->pQueryQ, pMsg); break; case FETCH_QUEUE: - dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pVnode->pWriteQ, pMsg); break; case SYNC_QUEUE: - dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pVnode->pSyncQ, pMsg); break; case MERGE_QUEUE: - dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pVnode->pMergeQ, pMsg); break; default: @@ -301,7 +300,7 @@ int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType qtype) { - SMsgHead * pHead = pRpc->pCont; + SMsgHead *pHead = pRpc->pCont; SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) return -1; @@ -469,7 +468,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "vnode-mgmt", - .fp = (FItem)vmProcessMgmtMonitorQueue, + .fp = (FItem)vmProcessQueue, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) { @@ -481,7 +480,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "vnode-monitor", - .fp = (FItem)vmProcessMgmtMonitorQueue, + .fp = (FItem)vmProcessQueue, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 5818b5880190f5db86e4e72557989544db012245..27f1140f2379f2db9a5856ff72ad0fbc0f42d9f2 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -134,10 +134,9 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType nType); int32_t dmMarkWrapper(SMgmtWrapper *pWrapper); void dmReleaseWrapper(SMgmtWrapper *pWrapper); SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper); - -void dmSetStatus(SDnode *pDnode, EDndRunStatus stype); -void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg); -void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg); +void dmSetStatus(SDnode *pDnode, EDndRunStatus stype); +void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg); +void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg); // dmNodes.c int32_t dmOpenNode(SMgmtWrapper *pWrapper); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 96285bbe1ca58bc1e3c900cb787aae0e3387e234..787f5e50190fb099d5998d4f604decd8b17feb68 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" #include "dmNodes.h" +#include "qworker.h" static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); @@ -279,38 +280,37 @@ static void dmGetServerStartupStatus(SDnode *pDnode, SServerStatusRsp *pStatus) void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg) { dDebug("msg:%p, net test req will be processed", pMsg); - SRpcMsg rsp = {.code = 0, .info = pMsg->info}; + + SRpcMsg rsp = {.info = pMsg->info}; rsp.pCont = rpcMallocCont(pMsg->contLen); if (rsp.pCont == NULL) { rsp.code = TSDB_CODE_OUT_OF_MEMORY; } else { rsp.contLen = pMsg->contLen; } + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); } void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg) { dDebug("msg:%p, server startup status req will be processed", pMsg); + SServerStatusRsp statusRsp = {0}; dmGetServerStartupStatus(pDnode, &statusRsp); - SRpcMsg rspMsg = {.info = pMsg->info}; - int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); - if (rspLen < 0) { - rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } - - void *pRsp = rpcMallocCont(rspLen); - if (pRsp == NULL) { - rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; + SRpcMsg rsp = {.info = pMsg->info}; + int32_t contLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); + if (contLen < 0) { + rsp.code = TSDB_CODE_OUT_OF_MEMORY; + } else { + rsp.pCont = rpcMallocCont(contLen); + if (rsp.pCont != NULL) { + tSerializeSServerStatusRsp(rsp.pCont, contLen, &statusRsp); + rsp.contLen = contLen; + } } - tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp); - rspMsg.pCont = pRsp; - rspMsg.contLen = rspLen; - -_OVER: - rpcSendResponse(&rspMsg); + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index d044ad3b2e5558c4c0660c12c4826068dceccbf1..72878d0d853e2067bdd73d42bb4b001ccfdb5295 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -142,13 +142,14 @@ static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg queue->tail = headLen + bodyLen; } else if (remain < 8 + headLen) { memcpy(queue->pBuffer + queue->tail + 8, pHead, remain - 8); - memcpy(queue->pBuffer, (char*)pHead + remain - 8, rawHeadLen - (remain - 8)); + memcpy(queue->pBuffer, (char *)pHead + remain - 8, rawHeadLen - (remain - 8)); if (rawBodyLen > 0) memcpy(queue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen); queue->tail = headLen - (remain - 8) + bodyLen; } else if (remain < 8 + headLen + bodyLen) { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, remain - 8 - headLen); - if (rawBodyLen > 0) memcpy(queue->pBuffer, (char*)pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); + if (rawBodyLen > 0) + memcpy(queue->pBuffer, (char *)pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); queue->tail = bodyLen - (remain - 8 - headLen); } else { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); @@ -312,12 +313,7 @@ static void *dmConsumChildQueue(void *param) { code = dmProcessNodeMsg(pWrapper, pMsg); if (code != 0) { dError("node:%s, failed to process msg:%p since %s, put into pqueue", proc->name, pMsg, terrstr()); - SRpcMsg rsp = { - .code = (terrno != 0 ? terrno : code), - .pCont = pMsg->info.rsp, - .contLen = pMsg->info.rspLen, - .info = pMsg->info, - }; + SRpcMsg rsp = {.code = (terrno != 0 ? terrno : code), .info = pMsg->info}; dmPutToProcPQueue(proc, &rsp, DND_FUNC_RSP); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); @@ -469,8 +465,18 @@ void dmPutToProcPQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype) { taosMsleep(retry); } } + + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + pMsg->contLen = 0; } int32_t dmPutToProcCQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype) { - return dmPushToProcQueue(proc, proc->cqueue, pMsg, ftype); + int32_t code = dmPushToProcQueue(proc, proc->cqueue, pMsg, ftype); + if (code == 0) { + dTrace("msg:%p, is freed after push to cqueue", pMsg); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } + return code; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index fe9a81bef1aefc698551b6aff54298b3e39826ff..6fbfae8b416efc68a0be9b101f1308aeba723752 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -17,9 +17,9 @@ #include "dmMgmt.h" #include "qworker.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" -#define INTERNAL_SECRET "_pwd" +static void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet); +static void dmSendRsp(SRpcMsg *pMsg); +static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg); static inline int32_t dmBuildNodeMsg(SRpcMsg *pMsg, SRpcMsg *pRpc) { SRpcConnInfo connInfo = {0}; @@ -49,49 +49,42 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { } static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { - SDnodeTrans * pTrans = &pDnode->trans; + SDnodeTrans *pTrans = &pDnode->trans; int32_t code = -1; - SRpcMsg * pMsg = NULL; - bool needRelease = false; - SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; + SRpcMsg *pMsg = NULL; SMgmtWrapper *pWrapper = NULL; + SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; dTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); - pRpc->info.noResp = 0; - pRpc->info.persistHandle = 0; - pRpc->info.wrapper = NULL; - pRpc->info.node = NULL; - pRpc->info.rsp = NULL; - pRpc->info.rspLen = 0; if (pRpc->msgType == TDMT_DND_NET_TEST) { dmProcessNetTestReq(pDnode, pRpc); - goto _OVER_JUST_FREE; + return; } else if (pRpc->msgType == TDMT_MND_SYSTABLE_RETRIEVE_RSP || pRpc->msgType == TDMT_VND_FETCH_RSP) { qWorkerProcessFetchRsp(NULL, NULL, pRpc); - goto _OVER_JUST_FREE; + return; } else { } if (pDnode->status != DND_STAT_RUNNING) { if (pRpc->msgType == TDMT_DND_SERVER_STATUS) { dmProcessServerStartupStatus(pDnode, pRpc); - goto _OVER_JUST_FREE; + return; } else { terrno = TSDB_CODE_APP_NOT_READY; - goto _OVER_RSP_FREE; + goto _OVER; } } if (IsReq(pRpc) && pRpc->pCont == NULL) { terrno = TSDB_CODE_INVALID_MSG_LEN; - goto _OVER_RSP_FREE; + goto _OVER; } if (pHandle->defaultNtype == NODE_END) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; - goto _OVER_RSP_FREE; + goto _OVER; } else { pWrapper = &pDnode->wrappers[pHandle->defaultNtype]; if (pHandle->needCheckVgId) { @@ -106,15 +99,15 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { } } else { terrno = TSDB_CODE_INVALID_MSG_LEN; - goto _OVER_RSP_FREE; + goto _OVER; } } } if (dmMarkWrapper(pWrapper) != 0) { - goto _OVER_RSP_FREE; + pWrapper = NULL; + goto _OVER; } else { - needRelease = true; pRpc->info.wrapper = pWrapper; } @@ -134,24 +127,23 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { } _OVER: - if (code == 0) { - if (pWrapper != NULL && InParentProc(pWrapper)) { - dTrace("msg:%p, is freed after push to cqueue", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - } else { + if (code != 0) { dError("msg:%p, failed to process since %s", pMsg, terrstr()); if (terrno != 0) code = terrno; if (IsReq(pRpc)) { - if (code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_NODE_OFFLINE) { - if (pRpc->msgType > TDMT_MND_MSG && pRpc->msgType < TDMT_VND_MSG) { - code = TSDB_CODE_NODE_REDIRECT; - } + SRpcMsg rsp = {.code = code, .info = pRpc->info}; + + if ((code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) && pRpc->msgType > TDMT_MND_MSG && + pRpc->msgType < TDMT_VND_MSG) { + dmBuildMnodeRedirectRsp(pDnode, &rsp); + } + + if (pWrapper != NULL) { + dmSendRsp(&rsp); + } else { + rpcSendResponse(&rsp); } - SRpcMsg rspMsg = {.code = code, .info = pRpc->info}; - tmsgSendRsp(&rspMsg); } dTrace("msg:%p, is freed", pMsg); @@ -159,19 +151,7 @@ _OVER: rpcFreeCont(pRpc->pCont); } - if (needRelease) { - dmReleaseWrapper(pWrapper); - } - return; - -_OVER_JUST_FREE: - rpcFreeCont(pRpc->pCont); - return; - -_OVER_RSP_FREE: - rpcFreeCont(pRpc->pCont); - SRpcMsg simpleRsp = {.code = terrno, .info = pRpc->info}; - rpcSendResponse(&simpleRsp); + dmReleaseWrapper(pWrapper); } int32_t dmInitMsgHandle(SDnode *pDnode) { @@ -179,11 +159,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; - SArray * pArray = (*pWrapper->func.getHandlesFp)(); + SArray *pArray = (*pWrapper->func.getHandlesFp)(); if (pArray == NULL) return -1; for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SMgmtHandle * pMgmt = taosArrayGet(pArray, i); + SMgmtHandle *pMgmt = taosArrayGet(pArray, i); SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)]; if (pMgmt->needCheckVgId) { pHandle->needCheckVgId = pMgmt->needCheckVgId; @@ -218,13 +198,25 @@ static inline void dmSendRsp(SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = pMsg->info.wrapper; if (InChildProc(pWrapper)) { dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_RSP); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; } else { rpcSendResponse(pMsg); } } +static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { + SMEpSet msg = {0}; + dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &msg.epSet); + + int32_t contLen = tSerializeSMEpSet(NULL, 0, &msg); + pMsg->pCont = rpcMallocCont(contLen); + if (pMsg->pCont == NULL) { + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + tSerializeSMEpSet(pMsg->pCont, contLen, &msg); + pMsg->contLen = contLen; + } +} + static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) { SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; SMEpSet msg = {.epSet = *pNewEpSet}; @@ -246,8 +238,6 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = pMsg->info.wrapper; if (InChildProc(pWrapper)) { dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_REGIST); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; } else { rpcRegisterBrokenLinkArg(pMsg); } @@ -275,7 +265,6 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = INTERNAL_USER; rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; @@ -343,34 +332,3 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) { }; return msgCb; } - -static void dmSendMnodeRedirectRsp(SRpcMsg *pMsg) { - SDnode *pDnode = dmInstance(); - SEpSet epSet = {0}; - dmGetMnodeEpSet(&pDnode->data, &epSet); - - dDebug("msg:%p, is redirected, num:%d use:%d", pMsg, epSet.numOfEps, epSet.inUse); - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); - if (strcmp(epSet.eps[i].fqdn, tsLocalFqdn) == 0 && epSet.eps[i].port == tsServerPort) { - epSet.inUse = (i + 1) % epSet.numOfEps; - } - - epSet.eps[i].port = htons(epSet.eps[i].port); - } - - SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; - SMEpSet msg = {.epSet = epSet}; - int32_t contLen = tSerializeSMEpSet(NULL, 0, &msg); - rsp.pCont = rpcMallocCont(contLen); - if (rsp.pCont == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - } else { - tSerializeSMEpSet(rsp.pCont, contLen, &msg); - rsp.contLen = contLen; - } - - dmSendRsp(&rsp); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; -} diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index e7256a3a87526d8c4b2b1e67189f8aa45935acf5..4946669678cd0fd17a22b935aa9e2613e58d73db 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -173,6 +173,7 @@ int32_t dmReadEps(SDnodeData *pData); int32_t dmWriteEps(SDnodeData *pData); void dmUpdateEps(SDnodeData *pData, SArray *pDnodeEps); void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); +void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet); void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); #ifdef __cplusplus diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 94fa5695578ccfbcdfc6b7a7c1d4ee834c1f99f9..e0af20e41bfef194d90d30316c16042522e7f87d 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -314,6 +314,17 @@ void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) { taosThreadRwlockUnlock(&pData->lock); } +void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet) { + dmGetMnodeEpSet(pData, pEpSet); + dDebug("msg:%p, is redirected, num:%d use:%d", pMsg, pEpSet->numOfEps, pEpSet->inUse); + for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { + dDebug("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + if (strcmp(pEpSet->eps[i].fqdn, tsLocalFqdn) == 0 && pEpSet->eps[i].port == tsServerPort) { + pEpSet->inUse = (i + 1) % pEpSet->numOfEps; + } + } +} + void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) { taosThreadRwlockWrlock(&pData->lock); pData->mnodeEps = *pEpSet; diff --git a/source/dnode/mgmt/test/bnode/dbnode.cpp b/source/dnode/mgmt/test/bnode/dbnode.cpp index 0568b30245d77e2a8dd7d2d227484477790b3270..c2a9873e5b0e4b2ddca8007e76d10e4e69fa3567 100644 --- a/source/dnode/mgmt/test/bnode/dbnode.cpp +++ b/source/dnode/mgmt/test/bnode/dbnode.cpp @@ -14,7 +14,7 @@ class DndTestBnode : public ::testing::Test { protected: static void SetUpTestSuite() { - test.Init("/tmp/dbnodeTest", 9112); + test.Init(TD_TMP_DIR_PATH "dbnodeTest", 9112); taosMsleep(1100); } static void TearDownTestSuite() { test.Cleanup(); } diff --git a/source/dnode/mgmt/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp index 98b50e96cfc84168ebecdb7d76ba23884c001d45..8c945b50ac48b4b1e290875c58a98f168971bc37 100644 --- a/source/dnode/mgmt/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/test/mnode/dmnode.cpp @@ -13,7 +13,7 @@ class DndTestMnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dmnodeTest", 9114); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dmnodeTest", 9114); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mgmt/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp index 2430419bef4e45e625ce60060849baa63eee3934..ef51be47a68d8772507dcee7ae8fae5b29d82bf6 100644 --- a/source/dnode/mgmt/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/test/qnode/dqnode.cpp @@ -13,7 +13,7 @@ class DndTestQnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dqnodeTest", 9111); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dqnodeTest", 9111); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mgmt/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp index 9ade616f191be10af24474fbf0f5581f30a1c061..9ae0fbdc542bb145c3b6e8d45fe355f1b187450d 100644 --- a/source/dnode/mgmt/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/test/snode/dsnode.cpp @@ -13,7 +13,7 @@ class DndTestSnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dsnodeTest", 9113); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dsnodeTest", 9113); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index 7bfa0417afbd957f81603371a3bb8f476868689e..6ef94481ea8edd66317f4d2a7b01bc3234ccd4d9 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -48,7 +48,7 @@ void Testbase::Init(const char* path, int16_t port) { strcpy(tsDataDir, path); taosRemoveDir(path); taosMkDir(path); - InitLog("/tmp/td"); + InitLog(TD_TMP_DIR_PATH "td"); server.Start(); client.Init("root", "taosdata"); diff --git a/source/dnode/mgmt/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp index bddf9518195d162c3934f8bb33fc34953f2f7830..8aba4f81b59ee35ff61616f0ada5056c8d76074a 100644 --- a/source/dnode/mgmt/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/test/vnode/vnode.cpp @@ -13,7 +13,7 @@ class DndTestVnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dvnodeTest", 9115); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dvnodeTest", 9115); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 924b6db8f7c57d18b6ab1fc47b43a785449a9eca..3316a09462ff1d5ff7c940e623941c7abe72a76c 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -304,10 +304,10 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { } code = mndCreateBnode(pMnode, pReq, pDnode, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("bnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); } @@ -414,10 +414,10 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { } code = mndDropBnode(pMnode, pReq, pObj); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("bnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 57f7b341d460ba52e7e44678d0397929ab320914..8b2799833b8acbfd658fb4e6bc034af92d6e415e 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -511,7 +511,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER; } - code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + code = TSDB_CODE_ACTION_IN_PROGRESS; SUBSCRIBE_OVER: mndTransDrop(pTrans); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 0bf7b240b26e545be633e2c6d923847e50bff5a1..6921235f8bfdb007811dc92ccff0514a68999824 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -525,7 +525,6 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; dbObj.cfg.pRetensions = pCreate->pRetensions; - pCreate->pRetensions = NULL; mndSetDefaultDbCfg(&dbObj.cfg); @@ -605,10 +604,10 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } code = mndCreateDb(pMnode, pReq, &createReq, pUser); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to create since %s", createReq.db, terrstr()); } @@ -839,10 +838,10 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndAlterDb(pMnode, pReq, pDb, &dbObj); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } @@ -1110,10 +1109,10 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { } code = mndDropDb(pMnode, pReq, pDb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to drop since %s", dropReq.db, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 2a4cf011156e777c4834a40a614d1e62a5b99465..01ff08cef9a8ebfd681038aea915fa7906d5ed12 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -504,10 +504,10 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { } code = mndCreateDnode(pMnode, pReq, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; CREATE_DNODE_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr()); } @@ -585,10 +585,10 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } code = mndDropDnode(pMnode, pReq, pDnode); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; DROP_DNODE_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index cf2edb57842c7eb0985ba8e3a27fe93e0bf28b5d..9107dab693d4c9eb6adc6599d03126d5a59a5a69 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -330,10 +330,10 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { } code = mndCreateFunc(pMnode, pReq, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("func:%s, failed to create since %s", createReq.name, terrstr()); } @@ -386,10 +386,10 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } code = mndDropFunc(pMnode, pReq, pFunc); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("func:%s, failed to drop since %s", dropReq.name, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 04c0a934858823cb7bc2d4a9e18637e72f9dfacf..7f86eb8b3292508c9b903c68fd6306b766ac074f 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -402,10 +402,10 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { } code = mndCreateMnode(pMnode, pReq, pDnode, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("mnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); } @@ -574,10 +574,10 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { } code = mndDropMnode(pMnode, pReq, pObj); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("mnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index b109ede81999e62b14bc7f95af486bdb830cdd33..6f42d66625aa52d4ce32b7fae3dd5947aea5fb1a 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -205,7 +205,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) { } mndTransDrop(pTrans); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return TSDB_CODE_ACTION_IN_PROGRESS; } static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index cd57e76b2cc6eacee39be027563d52d04a1ea038..b9ac82d890a12f2355834f415f4bbca65461873b 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -128,7 +128,8 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType } static void mndFreeConn(SConnObj *pConn) { - taosMemoryFreeClear(pConn->pQueries); + taosArrayDestroyEx(pConn->pQueries, tFreeClientHbQueryDesc); + mTrace("conn:%u, is destroyed, data:%p", pConn->id, pConn); } @@ -396,6 +397,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb if (NULL == hbRsp.info) { mError("taosArrayInit %d rsp kv failed", kvNum); terrno = TSDB_CODE_OUT_OF_MEMORY; + tFreeClientHbRsp(&hbRsp); return -1; } @@ -453,6 +455,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { SClientHbBatchReq batchReq = {0}; if (tDeserializeSClientHbBatchReq(pReq->pCont, pReq->contLen, &batchReq) != 0) { + taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq); terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -479,18 +482,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { void *buf = rpcMallocCont(tlen); tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp); - int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps); - for (int32_t i = 0; i < rspNum; ++i) { - SClientHbRsp *rsp = taosArrayGet(batchRsp.rsps, i); - int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info) : 0; - for (int32_t n = 0; n < kvNum; ++n) { - SKv *kv = taosArrayGet(rsp->info, n); - taosMemoryFreeClear(kv->value); - } - taosArrayDestroy(rsp->info); - } - - taosArrayDestroy(batchRsp.rsps); + tFreeClientHbBatchRsp(&batchRsp); pReq->info.rspLen = tlen; pReq->info.rsp = buf; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index c153d865523623f2e5350d12c98ce6d71e2516ba..3dc6200229b8a519fcf193393535500e98f4df20 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -306,10 +306,10 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { } code = mndCreateQnode(pMnode, pReq, pDnode, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("qnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); } @@ -416,10 +416,10 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { } code = mndDropQnode(pMnode, pReq, pObj); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("qnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 5c7089e1aa65f1de308f8bdfb770c866018f36ec..78b70c9a74133b859b4175b195d4a939c37ebccc 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -18,37 +18,35 @@ #include "mndMnode.h" #include "qworker.h" -int32_t mndProcessQueryMsg(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; +int32_t mndProcessQueryMsg(SRpcMsg *pMsg) { + int32_t code = -1; + SMnode *pMnode = pMsg->info.node; SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb}; - mTrace("msg:%p, in query queue is processing", pReq); - switch (pReq->msgType) { + mTrace("msg:%p, in query queue is processing", pMsg); + switch (pMsg->msgType) { case TDMT_VND_QUERY: - return qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pReq); + code = qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pMsg); + break; case TDMT_VND_QUERY_CONTINUE: - return qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pReq); - default: - mError("unknown msg type:%d in query queue", pReq->msgType); - return TSDB_CODE_VND_APP_ERROR; - } -} - -int32_t mndProcessFetchMsg(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - mTrace("msg:%p, in fetch queue is processing", pMsg); - - switch (pMsg->msgType) { + code = qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pMsg); + break; case TDMT_VND_FETCH: - return qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg); + code = qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg); + break; case TDMT_VND_DROP_TASK: - return qWorkerProcessDropMsg(pMnode, pMnode->pQuery, pMsg); + code = qWorkerProcessDropMsg(pMnode, pMnode->pQuery, pMsg); + break; case TDMT_VND_QUERY_HEARTBEAT: - return qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg); + code = qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg); + break; default: - mError("unknown msg type:%d in fetch queue", pMsg->msgType); - return TSDB_CODE_VND_APP_ERROR; + terrno = TSDB_CODE_VND_APP_ERROR; + mError("unknown msg type:%d in query queue", pMsg->msgType); } + + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + return code; } int32_t mndInitQuery(SMnode *pMnode) { @@ -59,9 +57,9 @@ int32_t mndInitQuery(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_QUERY, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_VND_QUERY_CONTINUE, mndProcessQueryMsg); - mndSetMsgHandle(pMnode, TDMT_VND_FETCH, mndProcessFetchMsg); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_TASK, mndProcessFetchMsg); - mndSetMsgHandle(pMnode, TDMT_VND_QUERY_HEARTBEAT, mndProcessFetchMsg); + mndSetMsgHandle(pMnode, TDMT_VND_FETCH, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_TASK, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_VND_QUERY_HEARTBEAT, mndProcessQueryMsg); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0f52f00d4e2059983fdf82cb3bde7fd16fae4b27..b38e901d49dee8e6c93dc629824e51bf44c71e0f 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -562,10 +562,10 @@ static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq) { } code = mndCreateSma(pMnode, pReq, &createReq, pDb, pStb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("sma:%s, failed to create since %s", createReq.name, terrstr(terrno)); } @@ -706,10 +706,10 @@ static int32_t mndProcessMDropSmaReq(SRpcMsg *pReq) { } code = mndDropSma(pMnode, pReq, pDb, pSma); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("sma:%s, failed to drop since %s", dropReq.name, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 5f58f2c8904f6958202d5decc4ffb693ee97070c..87b61f59ecb088692941a9f57ebf89db2cefa054 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -312,10 +312,10 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { } code = mndCreateSnode(pMnode, pReq, pDnode, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("snode:%d, failed to create since %s", createReq.dnodeId, terrstr()); return -1; } @@ -424,10 +424,10 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { } code = mndDropSnode(pMnode, pReq, pObj); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("snode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 7485510bc6f5db1873dced18e911adfe493a5d24..f6043615abea753db5e9a0bb7915932522eb7900 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -827,10 +827,10 @@ static int32_t mndProcessMCreateStbReq(SRpcMsg *pReq) { } code = mndCreateStb(pMnode, pReq, &createReq, pDb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stb:%s, failed to create since %s", createReq.name, terrstr()); } @@ -1334,10 +1334,10 @@ static int32_t mndProcessMAlterStbReq(SRpcMsg *pReq) { } code = mndAlterStb(pMnode, pReq, &alterReq, pDb, pStb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stb:%s, failed to alter since %s", alterReq.name, terrstr()); } @@ -1475,10 +1475,10 @@ static int32_t mndProcessMDropStbReq(SRpcMsg *pReq) { } code = mndDropStb(pMnode, pReq, pDb, pStb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); } @@ -1642,6 +1642,8 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int if (pStbVersion->sversion != metaRsp.sversion) { taosArrayPush(batchMetaRsp.pArray, &metaRsp); + } else { + tFreeSTableMetaRsp(&metaRsp); } } @@ -1660,6 +1662,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int } tSerializeSTableMetaBatchRsp(pRsp, rspLen, &batchMetaRsp); + tFreeSTableMetaBatchRsp(&batchMetaRsp); *ppRsp = pRsp; *pRspLen = rspLen; return 0; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 61a84fc95c335326a5f4172f66b4a1bd93ce1b41..7b6383a4704760a83e3520bdacbfb7b28efc7bd1 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -472,10 +472,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; CREATE_STREAM_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index c6eebb5c5d9600420806728066086699f6080b9b..b9b01a4391eeda3bcf277d4b85f315aa2accf65a 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -457,10 +457,10 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { } code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; CREATE_TOPIC_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); } @@ -547,7 +547,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return -1; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return TSDB_CODE_ACTION_IN_PROGRESS; } static int32_t mndProcessDropTopicInRsp(SRpcMsg *pRsp) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 35ecaa748ecdbdce486814f063b7678004f68909..5d205197d1734951451a54dac9758979ab4a8b04 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -994,9 +994,6 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgSent = 0; pAction->msgReceived = 0; pAction->errCode = terrno; - if (terrno == TSDB_CODE_INVALID_PTR || terrno == TSDB_CODE_NODE_OFFLINE) { - rpcFreeCont(rpcMsg.pCont); - } mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; } @@ -1038,13 +1035,13 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA } } else { mDebug("trans:%d, %d of %d actions executed", pTrans->id, numOfReceived, numOfActions); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return TSDB_CODE_ACTION_IN_PROGRESS; } } static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to execute redoActions since:%s, code:0x%x", terrstr(), terrno); } return code; @@ -1052,7 +1049,7 @@ static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to execute undoActions since %s", terrstr()); } return code; @@ -1091,7 +1088,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { pTrans->stage = TRN_STAGE_COMMIT; mDebug("trans:%d, stage from redoAction to commit", pTrans->id); continueExec = true; - } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, stage keep on redoAction since %s", pTrans->id, tstrerror(code)); continueExec = false; } else { @@ -1179,7 +1176,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { pTrans->stage = TRN_STAGE_UNDO_LOG; mDebug("trans:%d, stage from undoAction to undoLog", pTrans->id); continueExec = true; - } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, stage keep on undoAction since %s", pTrans->id, tstrerror(code)); continueExec = false; } else { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index f2a42e3083d0777eecb3ffd9db2a0359c7939ff6..88e646e76548bc0db5ed2af9e6e26ceb489c5cd0 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -331,10 +331,10 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { } code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("user:%s, failed to create since %s", createReq.user, terrstr()); } @@ -536,10 +536,10 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { } code = mndAlterUser(pMnode, pUser, &newUser, pReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("user:%s, failed to alter since %s", alterReq.user, terrstr()); } @@ -613,10 +613,10 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { } code = mndDropUser(pMnode, pReq, pUser); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("user:%s, failed to drop since %s", dropReq.user, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 8c805dd8c705d41f62a5728c8bf978d0f30924d5..285ae030e1424680f6af999b6554629d8b7a7653 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -367,7 +367,7 @@ int32_t mndProcessMsg(SRpcMsg *pMsg) { } int32_t code = (*fp)(pMsg); - if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code == TSDB_CODE_ACTION_IN_PROGRESS) { terrno = code; mTrace("msg:%p, in progress, app:%p", pMsg, ahandle); } else if (code != 0) { diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp index 6dcb931ed5e59c0a98904792cf25dfe762248ee7..46a9a465ebe631665fa753c6de17569160a158d1 100644 --- a/source/dnode/mnode/impl/test/acct/acct.cpp +++ b/source/dnode/mnode/impl/test/acct/acct.cpp @@ -13,7 +13,7 @@ class MndTestAcct : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/acctTest", 9012); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "acctTest", 9012); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/bnode/mbnode.cpp b/source/dnode/mnode/impl/test/bnode/mbnode.cpp index 316ac8cc3628e3b8cf747ab2e814c8dd6fcd50ff..c93e2142d02de318da323fa00ee7c04531d58874 100644 --- a/source/dnode/mnode/impl/test/bnode/mbnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/mbnode.cpp @@ -18,11 +18,11 @@ class MndTestBnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_bnode1", 9018); + test.Init(TD_TMP_DIR_PATH "mnode_test_bnode1", 9018); const char* fqdn = "localhost"; const char* firstEp = "localhost:9018"; - server2.Start("/tmp/mnode_test_bnode2", 9019); + server2.Start(TD_TMP_DIR_PATH "mnode_test_bnode2", 9019); taosMsleep(300); } diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 545f9f22bb869dbc432e81620425905ed89f73f6..a1bab5d1d4786bdcfba5774f657264f46e6f471c 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -13,7 +13,7 @@ class MndTestDb : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_db", 9030); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_db", 9030); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/dnode/mdnode.cpp b/source/dnode/mnode/impl/test/dnode/mdnode.cpp index e63536d4940a4779aa50a8947102eaf7c6904b34..0b42b2821988a4fe560d9d4660c8c70a3e9cbb4b 100644 --- a/source/dnode/mnode/impl/test/dnode/mdnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/mdnode.cpp @@ -18,14 +18,14 @@ class MndTestDnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/dnode_test_dnode1", 9023); + test.Init(TD_TMP_DIR_PATH "dnode_test_dnode1", 9023); const char* fqdn = "localhost"; const char* firstEp = "localhost:9023"; - // server2.Start("/tmp/dnode_test_dnode2", fqdn, 9024, firstEp); - // server3.Start("/tmp/dnode_test_dnode3", fqdn, 9025, firstEp); - // server4.Start("/tmp/dnode_test_dnode4", fqdn, 9026, firstEp); - // server5.Start("/tmp/dnode_test_dnode5", fqdn, 9027, firstEp); + // server2.Start(TD_TMP_DIR_PATH "dnode_test_dnode2", fqdn, 9024, firstEp); + // server3.Start(TD_TMP_DIR_PATH "dnode_test_dnode3", fqdn, 9025, firstEp); + // server4.Start(TD_TMP_DIR_PATH "dnode_test_dnode4", fqdn, 9026, firstEp); + // server5.Start(TD_TMP_DIR_PATH "dnode_test_dnode5", fqdn, 9027, firstEp); taosMsleep(300); } diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index c8f832160b4addc852fcf3f610dcd8d608cfef29..2bebe7ef199c8e61d1204a611b84b66051002087 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -13,7 +13,7 @@ class MndTestFunc : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_func", 9038); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_func", 9038); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/mnode/mnode.cpp b/source/dnode/mnode/impl/test/mnode/mnode.cpp index d953bdfdcba16c1987c2fb48980b2989b56bc400..1ed613c723ca504777929b8a6715e9d3b7594e06 100644 --- a/source/dnode/mnode/impl/test/mnode/mnode.cpp +++ b/source/dnode/mnode/impl/test/mnode/mnode.cpp @@ -18,11 +18,11 @@ class MndTestMnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_mnode1", 9028); + test.Init(TD_TMP_DIR_PATH "mnode_test_mnode1", 9028); const char* fqdn = "localhost"; const char* firstEp = "localhost:9028"; - // server2.Start("/tmp/mnode_test_mnode2", fqdn, 9029, firstEp); + // server2.Start(TD_TMP_DIR_PATH "mnode_test_mnode2", fqdn, 9029, firstEp); taosMsleep(300); } diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index 9c8e0298aa336ee5634480395a9e33ce3390d065..794374a91d51dfa9746028cebab6af3e394f2798 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -13,7 +13,7 @@ class MndTestProfile : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_profile", 9031); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_profile", 9031); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index 87ba7caa4edf888057aad3d3b1ab1e157e54b5e4..57b38e55c1d392bda809b094b0f1bf918aae8888 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -18,11 +18,11 @@ class MndTestQnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_qnode1", 9014); + test.Init(TD_TMP_DIR_PATH "mnode_test_qnode1", 9014); const char* fqdn = "localhost"; const char* firstEp = "localhost:9014"; - // server2.Start("/tmp/mnode_test_qnode2", fqdn, 9015, firstEp); + // server2.Start(TD_TMP_DIR_PATH "mnode_test_qnode2", fqdn, 9015, firstEp); taosMsleep(300); } diff --git a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp index b93adf99305492fec346a1fb981aef7e4e55979f..f67ca2fb91d1847ad1491802a0ee69052c90abed 100644 --- a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp +++ b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp @@ -31,7 +31,7 @@ class MndTestSdb : public ::testing::Test { tsLogEmbedded = 1; tsAsyncLog = 0; - const char *path = "/tmp/td"; + const char *path = TD_TMP_DIR_PATH "td"; taosRemoveDir(path); taosMkDir(path); tstrncpy(tsLogDir, path, PATH_MAX); @@ -385,7 +385,7 @@ TEST_F(MndTestSdb, 01_Write_Str) { mnode.v100 = 100; mnode.v200 = 200; opt.pMnode = &mnode; - opt.path = "/tmp/mnode_test_sdb"; + opt.path = TD_TMP_DIR_PATH "mnode_test_sdb"; taosRemoveDir(opt.path); SSdbTable strTable1; @@ -730,7 +730,7 @@ TEST_F(MndTestSdb, 01_Read_Str) { mnode.v100 = 100; mnode.v200 = 200; opt.pMnode = &mnode; - opt.path = "/tmp/mnode_test_sdb"; + opt.path = TD_TMP_DIR_PATH "mnode_test_sdb"; SSdbTable strTable1; memset(&strTable1, 0, sizeof(SSdbTable)); diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index 5c431f65d3a5eb5663eb3a92005a4635e6f7f384..0de8c9dca8cc69ce46bc41d439571fa8c7ad046b 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -13,7 +13,7 @@ class MndTestShow : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9021); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_show", 9021); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp index 4dc4e0477947d647961acd1859720f1013d6da45..d795816f57b5e65a7bedaef6c14af6db84680703 100644 --- a/source/dnode/mnode/impl/test/sma/sma.cpp +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -13,7 +13,7 @@ class MndTestSma : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_sma", 9035); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_sma", 9035); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/snode/snode.cpp b/source/dnode/mnode/impl/test/snode/snode.cpp index 0b1d3c38b25a204902e2b884c22006c89476dfb3..1828fbd570499107e3e3131631c35602bead983f 100644 --- a/source/dnode/mnode/impl/test/snode/snode.cpp +++ b/source/dnode/mnode/impl/test/snode/snode.cpp @@ -18,11 +18,11 @@ class MndTestSnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_snode1", 9016); + test.Init(TD_TMP_DIR_PATH "mnode_test_snode1", 9016); const char* fqdn = "localhost"; const char* firstEp = "localhost:9016"; - // server2.Start("/tmp/mnode_test_snode2", fqdn, 9017, firstEp); + // server2.Start(TD_TMP_DIR_PATH "mnode_test_snode2", fqdn, 9017, firstEp); taosMsleep(300); } diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index b8873210ab995bde06f6c2baf17d14975bc591e1..56f1b8240d0ee28a7023a204ed2e10ca9f93cc7e 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -13,7 +13,7 @@ class MndTestStb : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_stb", 9034); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_stb", 9034); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/topic/topic.cpp b/source/dnode/mnode/impl/test/topic/topic.cpp index eccc1b99d3f3dc8189712776e4ee852c6169cdfa..433a0ab5cc100d0fc96ac575a128f350df5e8bf3 100644 --- a/source/dnode/mnode/impl/test/topic/topic.cpp +++ b/source/dnode/mnode/impl/test/topic/topic.cpp @@ -13,7 +13,7 @@ class MndTestTopic : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_topic", 9039); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_topic", 9039); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/trans/trans1.cpp b/source/dnode/mnode/impl/test/trans/trans1.cpp index 80109a39b24c452f1a656b4bf172c8e2ad38acd5..5a470fc900f418c429befe12d6cfae643522a2e3 100644 --- a/source/dnode/mnode/impl/test/trans/trans1.cpp +++ b/source/dnode/mnode/impl/test/trans/trans1.cpp @@ -14,10 +14,10 @@ class MndTestTrans1 : public ::testing::Test { protected: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_trans1", 9013); + test.Init(TD_TMP_DIR_PATH "mnode_test_trans1", 9013); const char* fqdn = "localhost"; const char* firstEp = "localhost:9013"; - // server2.Start("/tmp/mnode_test_trans2", fqdn, 9020, firstEp); + // server2.Start(TD_TMP_DIR_PATH "mnode_test_trans2", fqdn, 9020, firstEp); } static void TearDownTestSuite() { @@ -26,7 +26,7 @@ class MndTestTrans1 : public ::testing::Test { } static void KillThenRestartServer() { - char file[PATH_MAX] = "/tmp/mnode_test_trans1/mnode/data/sdb.data"; + char file[PATH_MAX] = TD_TMP_DIR_PATH "mnode_test_trans1/mnode/data/sdb.data"; TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = taosMemoryMalloc(size); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index c4ed48fe60f069b05ca445c271a54d53bff810da..ec972cf784ce8f43e40def54b09d94eb13844c47 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -41,7 +41,7 @@ class MndTestTrans2 : public ::testing::Test { tsLogEmbedded = 1; tsAsyncLog = 0; - const char *logpath = "/tmp/td"; + const char *logpath = TD_TMP_DIR_PATH "td"; taosRemoveDir(logpath); taosMkDir(logpath); tstrncpy(tsLogDir, logpath, PATH_MAX); @@ -68,7 +68,7 @@ class MndTestTrans2 : public ::testing::Test { tsTransPullupInterval = 1; - const char *mnodepath = "/tmp/mnode_test_trans"; + const char *mnodepath = TD_TMP_DIR_PATH "mnode_test_trans"; taosRemoveDir(mnodepath); pMnode = mndOpen(mnodepath, &opt); mndStart(pMnode); diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 9e4bd79274a9da6810d1ab7f03edcd170d348183..6aa28a9007e8bc8e7c45621eb7e29280da964823 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -13,7 +13,7 @@ class MndTestUser : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_user", 9011); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_user", 9011); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 1259363f9468c2667536a17652010914dd4e2eac..929643fcdf91ef7ba0d6a02b8a07de34f0209d54 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -43,44 +43,49 @@ void qndClose(SQnode *pQnode) { int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { - qTrace("message in qnode query queue is processing"); + int32_t code = -1; SReadHandle handle = {.pMsgCb = &pQnode->msgCb}; + qTrace("message in qnode queue is processing"); switch (pMsg->msgType) { - case TDMT_VND_QUERY: { - return qWorkerProcessQueryMsg(&handle, pQnode->pQuery, pMsg); - } + case TDMT_VND_QUERY: + code = qWorkerProcessQueryMsg(&handle, pQnode->pQuery, pMsg); + break; case TDMT_VND_QUERY_CONTINUE: - return qWorkerProcessCQueryMsg(&handle, pQnode->pQuery, pMsg); - default: - qError("unknown msg type:%d in query queue", pMsg->msgType); - return TSDB_CODE_VND_APP_ERROR; - } -} - -int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { - qTrace("message in fetch queue is processing"); - switch (pMsg->msgType) { + code = qWorkerProcessCQueryMsg(&handle, pQnode->pQuery, pMsg); + break; case TDMT_VND_FETCH: - return qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_FETCH_RSP: - return qWorkerProcessFetchRsp(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessFetchRsp(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_RES_READY: - return qWorkerProcessReadyMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessReadyMsg(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_TASKS_STATUS: - return qWorkerProcessStatusMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessStatusMsg(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_CANCEL_TASK: - return qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_DROP_TASK: - return qWorkerProcessDropMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessDropMsg(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_TABLE_META: - // return vnodeGetTableMeta(pQnode, pMsg); + // code = vnodeGetTableMeta(pQnode, pMsg); + // break; case TDMT_VND_CONSUME: - // return tqProcessConsumeReq(pQnode->pTq, pMsg); + // code = tqProcessConsumeReq(pQnode->pTq, pMsg); + // break; case TDMT_VND_QUERY_HEARTBEAT: - return qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg); + break; default: - qError("unknown msg type:%d in fetch queue", pMsg->msgType); - return TSDB_CODE_VND_APP_ERROR; + qError("unknown msg type:%d in qnode queue", pMsg->msgType); + terrno = TSDB_CODE_VND_APP_ERROR; } + + if (code == 0) return TSDB_CODE_ACTION_IN_PROGRESS; + return code; } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a8e3860ed14035d5bedd5e1b096c45e1f03ec6e3..b10bee473284f272880ba94522065f65a9a58a7f 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -76,9 +76,14 @@ target_link_libraries( #PUBLIC scalar PUBLIC transport PUBLIC stream + PUBLIC index ) target_compile_definitions(vnode PUBLIC -DMETA_REFACT) - +if (${BUILD_WITH_INVERTEDINDEX}) + add_definitions(-DUSE_INVERTED_INDEX) +endif(${BUILD_WITH_INVERTEDINDEX}) if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) + + diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 693f4a0a2b4e0223f956990c59316b696b8a1c9a..3340bbb91ce0f8ed29b2ef48fc325472676b56e1 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -17,6 +17,7 @@ #define _TD_VNODE_META_H_ #include "vnodeInt.h" +#include "index.h" #ifdef __cplusplus extern "C" { @@ -61,16 +62,20 @@ static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64() struct SMeta { TdThreadRwlock lock; - char* path; - SVnode* pVnode; - TDB* pEnv; - TXN txn; - TTB* pTbDb; - TTB* pSkmDb; - TTB* pUidIdx; - TTB* pNameIdx; - TTB* pCtbIdx; - TTB* pTagIdx; + char* path; + SVnode* pVnode; + TDB* pEnv; + TXN txn; + TTB* pTbDb; + TTB* pSkmDb; + TTB* pUidIdx; + TTB* pNameIdx; + TTB* pCtbIdx; +#ifdef USE_INVERTED_INDEX + void* pTagIvtIdx; +#else + TTB* pTagIdx; +#endif TTB* pTtlIdx; TTB* pSmaIdx; SMetaIdx* pIdx; diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index b91622619fbb2675b24c028ce7065ce3b7c6096d..8a4db3100d31695af2889367088c9a0e16bb6236 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -56,8 +56,8 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeCStr(pCoder, &pME->name) < 0) return -1; if (pME->type == TSDB_SUPER_TABLE) { - if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schema) < 0) return -1; - if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schema) < 0) return -1; + if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; } else if (pME->type == TSDB_CHILD_TABLE) { if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; @@ -67,7 +67,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1; - if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; + if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schema) < 0) return -1; } else if (pME->type == TSDB_TSMA_TABLE) { pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma)); if (!pME->smaEntry.tsma) { diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 3f52071315bf15ca15beb2a14105c757d1b2eb25..efa06d2d1fea29995522e95b49789c6df3c2c435 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -53,10 +53,10 @@ int metaOpenIdx(SMeta *pMeta) { #endif #ifdef USE_INVERTED_INDEX - SIndexOpts opts; - if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { - return -1; - } + // SIndexOpts opts; + // if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { + // return -1; + //} #endif return 0; @@ -71,36 +71,37 @@ void metaCloseIdx(SMeta *pMeta) { /* TODO */ #endif #ifdef USE_INVERTED_INDEX - SIndexOpts opts; - if (indexClose(pMeta->pIdx->pIdx) != 0) { - return -1; - } + // SIndexOpts opts; + // if (indexClose(pMeta->pIdx->pIdx) != 0) { + // return -1; + //} + // return 0; #endif } int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) { #ifdef USE_INVERTED_INDEX - if (pTbCfgs->type == META_CHILD_TABLE) { - char buf[8] = {0}; - int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId; - sprintf(buf, "%d", colId); // colname - - char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); - - tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id - tb_uid_t tuid = 0; // child table uid - SIndexMultiTerm *terms = indexMultiTermCreate(); - SIndexTerm *term = - indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid); - indexMultiTermAdd(terms, term); - - int ret = indexPut(pMeta->pIdx->pIdx, terms); - indexMultiTermDestroy(terms); - return ret; - } else { - return DB_DONOTINDEX; - } + // if (pTbCfgs->type == META_CHILD_TABLE) { + // char buf[8] = {0}; + // int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId; + // sprintf(buf, "%d", colId); // colname + + // char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); + + // tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id + // tb_uid_t tuid = 0; // child table uid + // SIndexMultiTerm *terms = indexMultiTermCreate(); + // SIndexTerm *term = + // indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid); + // indexMultiTermAdd(terms, term); + + // int ret = indexPut(pMeta->pIdx->pIdx, terms); + // indexMultiTermDestroy(terms); + // return ret; + //} else { + // return DB_DONOTINDEX; + //} #endif // TODO return 0; @@ -112,4 +113,4 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) { #endif // TODO return 0; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 9a97357b97ac8e637d7a8cf72139b6615e7fdb05..f23e7f88056d6a397a5979bda11dd4f080ba0212 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -93,11 +93,24 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { } // open pTagIdx +#ifdef USE_INVERTED_INDEX + // TODO(yihaoDeng), refactor later + char indexFullPath[128] = {0}; + sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); + taosMkDir(indexFullPath); + ret = indexOpen(indexOptsCreate(), indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); + if (ret < 0) { + metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + +#else ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); if (ret < 0) { metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } +#endif // open pTtlIdx ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); @@ -128,7 +141,11 @@ _err: if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); +#ifdef USE_INVERTED_INDEX + if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); +#else if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); +#endif if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); @@ -145,7 +162,11 @@ int metaClose(SMeta *pMeta) { if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); +#ifdef USE_INVERTED_INDEX + if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); +#else if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); +#endif if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 2bcfd01904dbe32357e8f6afe1e81294ec9de50e..6cda56a5893e88a6801de71b357d995270484b58 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -30,9 +30,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int vLen = 0; const void *pKey = NULL; const void *pVal = NULL; - void *pBuf = NULL; + void * pBuf = NULL; int32_t szBuf = 0; - void *p = NULL; + void * p = NULL; SMetaReader mr = {0}; // validate req @@ -71,9 +71,9 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - TBC *pNameIdxc = NULL; - TBC *pUidIdxc = NULL; - TBC *pCtbIdxc = NULL; + TBC * pNameIdxc = NULL; + TBC * pUidIdxc = NULL; + TBC * pCtbIdxc = NULL; SCtbIdxKey *pCtbIdxKey; const void *pKey = NULL; int nKey; @@ -134,8 +134,8 @@ _err: int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; + TBC * pUidIdxc = NULL; + TBC * pTbDbc = NULL; const void *pData; int nData; int64_t oversion; @@ -256,9 +256,9 @@ _err: } int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) { - TBC *pTbDbc = NULL; - TBC *pUidIdxc = NULL; - TBC *pNameIdxc = NULL; + TBC * pTbDbc = NULL; + TBC * pUidIdxc = NULL; + TBC * pNameIdxc = NULL; const void *pData; int nData; tb_uid_t uid; @@ -377,14 +377,14 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi } static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - void *pVal = NULL; + void * pVal = NULL; int nVal = 0; - const void *pData = NULL; + const void * pData = NULL; int nData = 0; int ret = 0; tb_uid_t uid; int64_t oversion; - SSchema *pColumn = NULL; + SSchema * pColumn = NULL; SMetaEntry entry = {0}; SSchemaWrapper *pSchema; int c; @@ -483,12 +483,12 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS; goto _err; } - if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes <= pAlterTbReq->bytes) { + if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes > pAlterTbReq->colModBytes) { terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; goto _err; } pSchema->sver++; - pColumn->bytes = pAlterTbReq->bytes; + pColumn->bytes = pAlterTbReq->colModBytes; break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: if (pColumn == NULL) { @@ -530,7 +530,7 @@ _err: static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { SMetaEntry ctbEntry = {0}; SMetaEntry stbEntry = {0}; - void *pVal = NULL; + void * pVal = NULL; int nVal = 0; int ret; int c; @@ -561,8 +561,9 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA oversion = *(int64_t *)pData; // search table.db - TBC *pTbDbc = NULL; - SDecoder dc = {0}; + TBC * pTbDbc = NULL; + SDecoder dc1 = {0}; + SDecoder dc2 = {0}; /* get ctbEntry */ tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); @@ -572,21 +573,19 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ctbEntry.pBuf = taosMemoryMalloc(nData); memcpy(ctbEntry.pBuf, pData, nData); - tDecoderInit(&dc, ctbEntry.pBuf, nData); - metaDecodeEntry(&dc, &ctbEntry); - tDecoderClear(&dc); + tDecoderInit(&dc1, ctbEntry.pBuf, nData); + metaDecodeEntry(&dc1, &ctbEntry); /* get stbEntry*/ tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal); tdbFree(pVal); - tDecoderInit(&dc, stbEntry.pBuf, nVal); - metaDecodeEntry(&dc, &stbEntry); - tDecoderClear(&dc); + tDecoderInit(&dc2, stbEntry.pBuf, nVal); + metaDecodeEntry(&dc2, &stbEntry); SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema *pColumn = NULL; + SSchema * pColumn = NULL; int32_t iCol = 0; for (;;) { pColumn = NULL; @@ -638,6 +637,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA // save to uid.idx tdbTbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); + tDecoderClear(&dc1); + tDecoderClear(&dc2); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); tdbTbcClose(pTbDbc); @@ -645,6 +646,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA return 0; _err: + tDecoderClear(&dc1); + tDecoderClear(&dc2); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); tdbTbcClose(pTbDbc); @@ -678,8 +681,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { STbDbKey tbDbKey; - void *pKey = NULL; - void *pVal = NULL; + void * pKey = NULL; + void * pVal = NULL; int kLen = 0; int vLen = 0; SEncoder coder = {0}; @@ -794,14 +797,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { } static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { - void *pData = NULL; + void * pData = NULL; int nData = 0; STbDbKey tbDbKey = {0}; SMetaEntry stbEntry = {0}; - STagIdxKey *pTagIdxKey = NULL; + STagIdxKey * pTagIdxKey = NULL; int32_t nTagIdxKey; const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; - const void *pTagData = NULL; // + const void * pTagData = NULL; // SDecoder dc = {0}; // get super table @@ -817,22 +820,33 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { pTagData = tdGetKVRowValOfCol((const SKVRow)pCtbEntry->ctbEntry.pTags, pTagColumn->colId); // update tag index +#ifdef USE_INVERTED_INDEX + tb_uid_t suid = pCtbEntry->ctbEntry.suid; + tb_uid_t tuid = pCtbEntry->uid; + + SIndexMultiTerm *tmGroup = indexMultiTermCreate(); + + SIndexTerm *tm = indexTermCreate(suid, ADD_VALUE, pTagColumn->type, pTagColumn->name, sizeof(pTagColumn->name), + pTagData, pTagData == NULL ? 0 : strlen(pTagData)); + indexMultiTermAdd(tmGroup, tm); + int ret = indexPut((SIndex *)pMeta->pTagIvtIdx, tmGroup, tuid); + indexMultiTermDestroy(tmGroup); +#else if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, pTagColumn->type, pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { return -1; } tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); - +#endif tDecoderClear(&dc); tdbFree(pData); - return 0; } static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { SEncoder coder = {0}; - void *pVal = NULL; + void * pVal = NULL; int vLen = 0; int rcode = 0; SSkmDbKey skmDbKey = {0}; diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 1c7db28e189d918fb68e81dabb6249ea246e2bea..2a74fe78cbc66a3873857347df010190554e1e76 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -132,6 +132,7 @@ int32_t smaClose(SSma *pSma) { if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma)); if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma)); if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma)); + taosMemoryFree(pSma); } return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1f35ec2650f2cc5c6640e6c92bf308bd162663f9..25fa716d4e386b596d2777b996d2f3cc09cc20e1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -57,6 +57,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { void tqClose(STQ* pTq) { if (pTq) { taosMemoryFreeClear(pTq->path); + taosHashCleanup(pTq->execs); + taosHashCleanup(pTq->pStreamTasks); + taosHashCleanup(pTq->pushMgr); taosMemoryFree(pTq); } // TODO @@ -409,9 +412,9 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu pTopic->buffer.output[j].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { - .reader = pReadHandle, - .meta = pTq->pVnode->pMeta, - .pMsgCb = &pTq->pVnode->msgCb, + .reader = pReadHandle, + .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, }; pTopic->buffer.output[j].pReadHandle = pReadHandle; pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); @@ -597,6 +600,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SSDataBlock block = {0}; if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, &block.info.numOfCols) < 0) { + if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block); @@ -1000,10 +1004,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { for (int32_t i = 0; i < parallel; i++) { STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { - .reader = pStreamReader, - .meta = pTq->pVnode->pMeta, - .pMsgCb = &pTq->pVnode->msgCb, - .vnode = pTq->pVnode, + .reader = pStreamReader, + .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, + .vnode = pTq->pVnode, }; pTask->exec.runners[i].inputHandle = pStreamReader; pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 096208b96196dd3a83089e8c027d576f408f0270..be8d786de2e7c015f938e87431129f5b2d067d00 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -90,9 +90,23 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t sversion = 1; if (pHandle->sver != sversion || pHandle->cachedSchemaUid != pHandle->msgIter.suid) { pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); + if (pHandle->pSchema == NULL) { + tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table", + pHandle->msgIter.uid, pHandle->msgIter.suid, pHandle->sver); + /*ASSERT(0);*/ + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } // this interface use suid instead of uid pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.suid, sversion, true); + if (pHandle->pSchemaWrapper == NULL) { + tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table", + pHandle->msgIter.suid, pHandle->sver); + /*ASSERT(0);*/ + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } pHandle->sver = sversion; pHandle->cachedSchemaUid = pHandle->msgIter.suid; } @@ -190,7 +204,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p } return 0; FAIL: - taosArrayDestroy(*ppCols); + if (*ppCols) taosArrayDestroy(*ppCols); return -1; } @@ -235,8 +249,8 @@ int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { int tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { ASSERT(pHandle->tbIdHash != NULL); - for(int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t* pKey = (int64_t*) taosArrayGet(tbUidList, i); + for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { + int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); taosHashRemove(pHandle->tbIdHash, pKey, sizeof(int64_t)); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 76d5c3cb3a9e869a62ae0df3222d1e66ff3ac646..93ec6028f86e10dfcf91db7a79ed03e64d2f55db 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -477,6 +477,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pCommitIter->pTable->pSchema = pTSchema; // metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); } } + tSkipListDestroyIter(pSlIter); return 0; } @@ -1137,6 +1138,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF memcpy(tptr, pDataCol->pData, flen); if (tBitmaps > 0) { bptr = POINTER_SHIFT(pBlockData, lsize + flen); + if (isSuper && !tdDataColsIsBitmapI(pDataCols)) { + tdMergeBitmap((uint8_t *)pDataCol->pBitmap, rowsToWrite, (uint8_t *)pDataCol->pBitmap); + } memcpy(bptr, pDataCol->pBitmap, tBitmaps); tBitmapsLen = tBitmaps; flen += tBitmapsLen; @@ -1502,13 +1506,16 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt tSkipListIterNext(pCommitIter->pIter); } else { if (lastKey != key1) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; + } lastKey = key1; - ++pTarget->numOfRows; } // copy disk data for (int i = 0; i < pDataCols->numOfCols; ++i) { SCellVal sVal = {0}; + // no duplicated TS keys in pDataCols from file if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) { TASSERT(0); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f0aa4d2ac64fb1d5931fb768b7def18ec9f58292..47896272b5564a76c5aa206dc7022cfb9bf13cc5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1732,6 +1732,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } + *lastRowKey = rowKey; ++nResult; } else if (update) { mergeOption = 2; @@ -1739,8 +1740,6 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa mergeOption = 0; break; } - - *lastRowKey = rowKey; } } else { // TODO: use STSRowIter @@ -1753,6 +1752,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } + *lastRowKey = rowKey; ++nResult; } else if (update) { mergeOption = 2; @@ -1760,7 +1760,6 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa mergeOption = 0; break; } - *lastRowKey = rowKey; } else { SKvRowIdx* pColIdx = tdKvRowColIdxAt(row, chosen_itr - 1); colId = pColIdx->colId; @@ -1965,7 +1964,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID && cur->pos >= 0 && cur->pos < pBlock->numOfRows); - + // Even Multi-Version supported, the records with duplicated TSKEY would be merged inside of tsdbLoadData interface. TSKEY* tsArray = pCols->cols[0].pData; assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows - 1] == pBlock->keyLast); @@ -1995,6 +1994,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf int32_t pos = cur->pos; cur->win = TSWINDOW_INITIALIZER; + bool adjustPos = false; // no data in buffer, load data from file directly if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) { @@ -2016,6 +2016,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf break; } + if (adjustPos) { + if (key == lastKeyAppend) { + pos -= step; + } + adjustPos = false; + } + if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ascScan) || ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ascScan)) { break; @@ -2107,7 +2114,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf moveToNextRowInMem(pCheckInfo); pos += step; + adjustPos = true; } else { + // discard the memory record moveToNextRowInMem(pCheckInfo); } } else if ((key > tsArray[pos] && ascScan) || (key < tsArray[pos] && !ascScan)) { @@ -2775,7 +2784,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int win->ekey = key; if (rv != TD_ROW_SVER(row)) { - pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 1); + pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, TD_ROW_SVER(row)); rv = TD_ROW_SVER(row); } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index c1a1e7570ecdeba56c03f9b1b6fcb28894089610..0773805684a01c67324143d09b9a2fbdee663bb7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -20,11 +20,11 @@ static void tsdbResetReadTable(SReadH *pReadh); static void tsdbResetReadFile(SReadH *pReadh); static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock); -static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols); +static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int8_t bitmapMode); static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp, int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize); static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds, - int numOfColIds); + int numOfColIds, int8_t bitmapMode); static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol); int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo) { @@ -266,10 +266,11 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { } } - if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0]) < 0) return -1; + + if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0], TSDB_BITMODE_ONE_BIT) < 0) return -1; for (int i = 1; i < pBlock->numOfSubBlocks; i++) { iBlock++; - if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1; + if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1], TSDB_BITMODE_DEFAULT) < 0) return -1; // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) @@ -309,10 +310,10 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, } } - if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds) < 0) return -1; + if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds, TSDB_BITMODE_ONE_BIT) < 0) return -1; for (int i = 1; i < pBlock->numOfSubBlocks; i++) { iBlock++; - if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1; + if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds, TSDB_BITMODE_DEFAULT) < 0) return -1; // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) @@ -543,14 +544,14 @@ static void tsdbResetReadFile(SReadH *pReadh) { tsdbCloseDFileSet(TSDB_READ_FSET(pReadh)); } -static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols) { +static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int8_t bitmapMode) { ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); tdResetDataCols(pDataCols); - if (tsdbIsSupBlock(pBlock)) { + if (tdIsBitmapModeI(bitmapMode)) { tdDataColsSetBitmapI(pDataCols); } @@ -730,7 +731,7 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 } static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds, - int numOfColIds) { + int numOfColIds, int8_t bitmapMode) { ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID); @@ -739,7 +740,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * tdResetDataCols(pDataCols); - if (tsdbIsSupBlock(pBlock)) { + if (tdIsBitmapModeI(bitmapMode)) { tdDataColsSetBitmapI(pDataCols); } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index f0af677641c3e08cd4d848ec4c98b28fce4662ee..a90bb7afcb6847ba0cb803d7a7e58720159bf10f 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -77,7 +77,8 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode->path = (char *)&pVnode[1]; strcpy(pVnode->path, path); pVnode->config = info.config; - pVnode->state = info.state; + pVnode->state.committed = info.state.committed; + pVnode->state.applied = info.state.committed; pVnode->pTfs = pTfs; pVnode->msgCb = msgCb; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 297b518ac76d215b40c5527a2822dd9bf48acba6..88cc97ddd525ed20d47c3c82879fad2d792953be 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -147,9 +147,6 @@ _err: int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); -#if 0 - SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; -#endif SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; switch (pMsg->msgType) { case TDMT_VND_QUERY: diff --git a/source/dnode/vnode/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp index 627dbc6f18122e39cde2e243fd564ad622324cd2..168daeb19fc4ab90d502b7694d2c54a9ec102c8c 100644 --- a/source/dnode/vnode/test/tqMetaTest.cpp +++ b/source/dnode/vnode/test/tqMetaTest.cpp @@ -41,7 +41,7 @@ class TqMetaUpdateAppendTest : public ::testing::Test { void TearDown() override { tqStoreClose(pMeta); } STqMetaStore* pMeta; - const char* pathName = "/tmp/tq_test"; + const char* pathName = TD_TMP_DIR_PATH "tq_test"; }; TEST_F(TqMetaUpdateAppendTest, copyPutTest) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d5301e3624fd632d31e48e6eae0944d5ceebd8be..2d4355191ef73d801cc7718f45ea8191b1aa7232 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "trpc.h" -#include "query.h" -#include "tname.h" #include "catalogInt.h" +#include "query.h" #include "systable.h" +#include "tname.h" +#include "trpc.h" int32_t ctgActUpdateVg(SCtgMetaAction *action); int32_t ctgActUpdateTbl(SCtgMetaAction *action); @@ -27,38 +27,11 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action); int32_t ctgActUpdateUser(SCtgMetaAction *action); extern SCtgDebug gCTGDebug; -SCatalogMgmt gCtgMgmt = {0}; -SCtgAction gCtgAction[CTG_ACT_MAX] = {{ - CTG_ACT_UPDATE_VG, - "update vgInfo", - ctgActUpdateVg - }, - { - CTG_ACT_UPDATE_TBL, - "update tbMeta", - ctgActUpdateTbl - }, - { - CTG_ACT_REMOVE_DB, - "remove DB", - ctgActRemoveDB - }, - { - CTG_ACT_REMOVE_STB, - "remove stbMeta", - ctgActRemoveStb - }, - { - CTG_ACT_REMOVE_TBL, - "remove tbMeta", - ctgActRemoveTbl - }, - { - CTG_ACT_UPDATE_USER, - "update user", - ctgActUpdateUser - } -}; +SCatalogMgmt gCtgMgmt = {0}; +SCtgAction gCtgAction[CTG_ACT_MAX] = { + {CTG_ACT_UPDATE_VG, "update vgInfo", ctgActUpdateVg}, {CTG_ACT_UPDATE_TBL, "update tbMeta", ctgActUpdateTbl}, + {CTG_ACT_REMOVE_DB, "remove DB", ctgActRemoveDB}, {CTG_ACT_REMOVE_STB, "remove stbMeta", ctgActRemoveStb}, + {CTG_ACT_REMOVE_TBL, "remove tbMeta", ctgActRemoveTbl}, {CTG_ACT_UPDATE_USER, "update user", ctgActUpdateUser}}; void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { if (NULL == mgmt->slots) { @@ -76,11 +49,10 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { taosMemoryFreeClear(mgmt->slots); } - void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { - int32_t stblNum = taosHashGetSize(cache->stbCache); + int32_t stblNum = taosHashGetSize(cache->stbCache); taosHashCleanup(cache->stbCache); cache->stbCache = NULL; CTG_CACHE_STAT_SUB(stblNum, stblNum); @@ -106,7 +78,7 @@ void ctgFreeVgInfo(SDBVgInfo *vgInfo) { taosHashCleanup(vgInfo->vgHash); vgInfo->vgHash = NULL; } - + taosMemoryFreeClear(vgInfo); } @@ -116,7 +88,7 @@ void ctgFreeDbCache(SCtgDBCache *dbCache) { } CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - ctgFreeVgInfo (dbCache->vgInfo); + ctgFreeVgInfo(dbCache->vgInfo); CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); ctgFreeTableMetaCache(&dbCache->tbCache); @@ -128,13 +100,13 @@ void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) { taosHashCleanup(userCache->writeDbs); } -void ctgFreeHandle(SCatalog* pCtg) { +void ctgFreeHandle(SCatalog *pCtg) { ctgFreeMetaRent(&pCtg->dbRent); ctgFreeMetaRent(&pCtg->stbRent); - + if (pCtg->dbCache) { int32_t dbNum = taosHashGetSize(pCtg->dbCache); - + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; @@ -142,42 +114,40 @@ void ctgFreeHandle(SCatalog* pCtg) { atomic_store_8(&dbCache->deleted, 1); ctgFreeDbCache(dbCache); - + pIter = taosHashIterate(pCtg->dbCache, pIter); - } + } taosHashCleanup(pCtg->dbCache); - + CTG_CACHE_STAT_SUB(dbNum, dbNum); } if (pCtg->userCache) { int32_t userNum = taosHashGetSize(pCtg->userCache); - + void *pIter = taosHashIterate(pCtg->userCache, NULL); while (pIter) { SCtgUserAuth *userCache = pIter; ctgFreeSCtgUserAuth(userCache); - + pIter = taosHashIterate(pCtg->userCache, pIter); - } + } taosHashCleanup(pCtg->userCache); - + CTG_CACHE_STAT_SUB(userNum, userNum); } - + taosMemoryFree(pCtg); } - - void ctgWaitAction(SCtgMetaAction *action) { while (true) { tsem_wait(&gCtgMgmt.queue.rspSem); - - if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + + if (atomic_load_8((int8_t *)&gCtgMgmt.exit)) { tsem_post(&gCtgMgmt.queue.rspSem); break; } @@ -193,19 +163,18 @@ void ctgWaitAction(SCtgMetaAction *action) { void ctgPopAction(SCtgMetaAction **action) { SCtgQNode *orig = gCtgMgmt.queue.head; - + SCtgQNode *node = gCtgMgmt.queue.head->next; gCtgMgmt.queue.head = gCtgMgmt.queue.head->next; CTG_QUEUE_SUB(); - + taosMemoryFreeClear(orig); *action = &node->action; } - -int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { +int32_t ctgPushAction(SCatalog *pCtg, SCtgMetaAction *action) { SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); @@ -213,7 +182,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { } action->seqId = atomic_add_fetch_64(&gCtgMgmt.queue.seqId, 1); - + node->action = *action; CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); @@ -235,10 +204,9 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { return TSDB_CODE_SUCCESS; } - -int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; +int32_t ctgPushRmDBMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_REMOVE_DB}; SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); @@ -266,10 +234,10 @@ _return: CTG_RET(code); } - -int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq}; +int32_t ctgPushRmStbMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, + bool syncReq) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq}; SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); @@ -294,11 +262,9 @@ _return: CTG_RET(code); } - - -int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq}; +int32_t ctgPushRmTblMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq}; SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); @@ -322,9 +288,9 @@ _return: CTG_RET(code); } -int32_t ctgPushUpdateVgMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; +int32_t ctgPushUpdateVgMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDBVgInfo *dbInfo, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); @@ -355,9 +321,9 @@ _return: CTG_RET(code); } -int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq}; +int32_t ctgPushUpdateTblMsgInQueue(SCatalog *pCtg, STableMetaOutput *output, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq}; SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); @@ -377,17 +343,17 @@ int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, boo CTG_ERR_JRET(ctgPushAction(pCtg, &action)); return TSDB_CODE_SUCCESS; - + _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } -int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; +int32_t ctgPushUpdateUserMsgInQueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); @@ -402,38 +368,37 @@ int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool CTG_ERR_JRET(ctgPushAction(pCtg, &action)); return TSDB_CODE_SUCCESS; - + _return: tFreeSGetUserAuthRsp(pAuth); taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgLock); - + if (dbCache->deleted) { CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); - + ctgDebug("db is dropping, dbId:%" PRIx64, dbCache->dbId); + *inCache = false; return TSDB_CODE_SUCCESS; } - if (NULL == dbCache->vgInfo) { CTG_UNLOCK(CTG_READ, &dbCache->vgLock); *inCache = false; - ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); + ctgDebug("db vgInfo is empty, dbId:%" PRIx64, dbCache->dbId); return TSDB_CODE_SUCCESS; } *inCache = true; - + return TSDB_CODE_SUCCESS; } @@ -441,7 +406,7 @@ int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { CTG_LOCK(CTG_WRITE, &dbCache->vgLock); if (dbCache->deleted) { - ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + ctgDebug("db is dropping, dbId:%" PRIx64, dbCache->dbId); CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } @@ -449,20 +414,13 @@ int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { return TSDB_CODE_SUCCESS; } -void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { - taosHashRelease(pCtg->dbCache, dbCache); -} - -void ctgReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); -} +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { taosHashRelease(pCtg->dbCache, dbCache); } -void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); -} +void ctgReleaseVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgLock); } +void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); } -int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { +int32_t ctgAcquireDBCacheImpl(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { char *p = strchr(dbFName, '.'); if (p && CTG_IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; @@ -474,7 +432,7 @@ int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache * } else { dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); } - + if (NULL == dbCache) { *pCache = NULL; ctgDebug("db not in cache, dbFName:%s", dbFName); @@ -484,28 +442,27 @@ int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache * if (dbCache->deleted) { if (acquire) { ctgReleaseDBCache(pCtg, dbCache); - } - + } + *pCache = NULL; ctgDebug("db is removing from cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } *pCache = dbCache; - + return TSDB_CODE_SUCCESS; } -int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { +int32_t ctgAcquireDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); } -int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { +int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } - -int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { +int32_t ctgAcquireVgInfoFromCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { SCtgDBCache *dbCache = NULL; if (NULL == pCtg->dbCache) { @@ -514,11 +471,11 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac } ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { + if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; } - + ctgAcquireVgInfo(pCtg, dbCache, inCache); if (!(*inCache)) { ctgDebug("vgInfo of db %s not in cache", dbFName); @@ -531,7 +488,7 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac CTG_CACHE_STAT_ADD(vgHitNum, 1); ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); - + return TSDB_CODE_SUCCESS; _return: @@ -544,12 +501,12 @@ _return: *inCache = false; CTG_CACHE_STAT_ADD(vgMissNum, 1); - + return TSDB_CODE_SUCCESS; } -int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray *out) { - char *msg = NULL; +int32_t ctgGetQnodeListFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SArray *out) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pMgmtEps->inUse); @@ -559,16 +516,16 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt ctgError("Build qnode list msg failed, error:%s", tstrerror(code)); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_QNODE_LIST, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { ctgError("error rsp for qnode list, error:%s", tstrerror(rpcRsp.code)); CTG_ERR_RET(rpcRsp.code); @@ -585,9 +542,9 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt return TSDB_CODE_SUCCESS; } - -int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { - char *msg = NULL; +int32_t ctgGetDBVgInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SBuildUseDBInput *input, + SUseDbOutput *out) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); @@ -597,16 +554,16 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtE ctgError("Build use db msg failed, code:%x, db:%s", code, input->db); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_USE_DB, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { ctgError("error rsp for use db, error:%s, db:%s", tstrerror(rpcRsp.code), input->db); CTG_ERR_RET(rpcRsp.code); @@ -623,8 +580,8 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtE return TSDB_CODE_SUCCESS; } -int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *dbFName, SDbCfgInfo *out) { - char *msg = NULL; +int32_t ctgGetDBCfgFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SDbCfgInfo *out) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName); @@ -634,16 +591,16 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, ctgError("Build get db cfg msg failed, code:%x, db:%s", code, dbFName); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_GET_DB_CFG, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { ctgError("error rsp for get db cfg, error:%s, db:%s", tstrerror(rpcRsp.code), dbFName); CTG_ERR_RET(rpcRsp.code); @@ -660,8 +617,9 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, return TSDB_CODE_SUCCESS; } -int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *indexName, SIndexInfo *out) { - char *msg = NULL; +int32_t ctgGetIndexInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *indexName, + SIndexInfo *out) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get index from mnode, indexName:%s", indexName); @@ -671,16 +629,16 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt ctgError("Build get index msg failed, code:%x, db:%s", code, indexName); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_GET_INDEX, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { ctgError("error rsp for get index, error:%s, indexName:%s", tstrerror(rpcRsp.code), indexName); CTG_ERR_RET(rpcRsp.code); @@ -697,8 +655,9 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt return TSDB_CODE_SUCCESS; } -int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *funcName, SFuncInfo **out) { - char *msg = NULL; +int32_t ctgGetUdfInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *funcName, + SFuncInfo **out) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get udf info from mnode, funcName:%s", funcName); @@ -708,23 +667,23 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEp ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_RETRIEVE_FUNC, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (TSDB_CODE_MND_FUNC_NOT_EXIST == rpcRsp.code) { ctgDebug("funcName %s not exist in mnode", funcName); taosMemoryFreeClear(*out); CTG_RET(TSDB_CODE_SUCCESS); } - + ctgError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rpcRsp.code), funcName); CTG_ERR_RET(rpcRsp.code); } @@ -740,8 +699,9 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEp return TSDB_CODE_SUCCESS; } -int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *user, SGetUserAuthRsp *authRsp) { - char *msg = NULL; +int32_t ctgGetUserDbAuthFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, + SGetUserAuthRsp *authRsp) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get user auth from mnode, user:%s", user); @@ -751,16 +711,16 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_GET_USER_AUTH, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { ctgError("error rsp for get user auth, error:%s, user:%s", tstrerror(rpcRsp.code), user); CTG_ERR_RET(rpcRsp.code); @@ -777,8 +737,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm return TSDB_CODE_SUCCESS; } - -int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { +int32_t ctgIsTableMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32_t *exist) { if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); @@ -793,13 +752,13 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, } size_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - + if (NULL == tbMeta) { ctgReleaseDBCache(pCtg, dbCache); - + *exist = 0; ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; @@ -808,14 +767,14 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, *exist = 1; ctgReleaseDBCache(pCtg, dbCache); - + ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); - + return TSDB_CODE_SUCCESS; } - -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, bool *inCache, int32_t flag, uint64_t *dbId) { +int32_t ctgGetTableMetaFromCache(SCatalog *pCtg, const SName *pTableName, STableMeta **pTableMeta, bool *inCache, + int32_t flag, uint64_t *dbId) { if (NULL == pCtg->dbCache) { ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); goto _return; @@ -836,10 +795,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable ctgDebug("db %s not in cache", pTableName->tname); goto _return; } - + int32_t sz = 0; CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), (void **)pTableMeta, &sz); + int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), + (void **)pTableMeta, &sz); CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == *pTableMeta) { @@ -852,7 +812,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable *dbId = dbCache->dbId; } - STableMeta* tbMeta = *pTableMeta; + STableMeta *tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); @@ -860,34 +820,36 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable *inCache = true; CTG_CACHE_STAT_ADD(tblHitNum, 1); - + return TSDB_CODE_SUCCESS; } - ctgDebug("Got subtable meta from cache, type:%d, dbFName:%s, tbName:%s, suid:%" PRIx64, tbMeta->tableType, dbFName, pTableName->tname, tbMeta->suid); - + ctgDebug("Got subtable meta from cache, type:%d, dbFName:%s, tbName:%s, suid:%" PRIx64, tbMeta->tableType, dbFName, + pTableName->tname, tbMeta->suid); + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); - ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); + ctgError("stb not in stbCache, suid:%" PRIx64, tbMeta->suid); taosMemoryFreeClear(*pTableMeta); goto _return; } - if ((*stbMeta)->suid != tbMeta->suid) { + if ((*stbMeta)->suid != tbMeta->suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); taosMemoryFreeClear(*pTableMeta); - ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, tbMeta->suid, + (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = CTG_META_SIZE(*stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); - if (NULL == *pTableMeta) { + if (NULL == *pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); ctgError("realloc size[%d] failed", metaSize); @@ -904,23 +866,23 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable CTG_CACHE_STAT_ADD(tblHitNum, 1); ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); - + return TSDB_CODE_SUCCESS; _return: *inCache = false; CTG_CACHE_STAT_ADD(tblMissNum, 1); - + return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { +int32_t ctgGetTableTypeFromCache(SCatalog *pCtg, const char *dbFName, const char *tableName, int32_t *tbType) { if (NULL == pCtg->dbCache) { - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); + ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); return TSDB_CODE_SUCCESS; } - + SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { @@ -932,9 +894,9 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char if (NULL == pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); + ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); ctgReleaseDBCache(pCtg, dbCache); - + return TSDB_CODE_SUCCESS; } @@ -946,17 +908,18 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); - + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); + return TSDB_CODE_SUCCESS; } -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { +int32_t ctgChkAuthFromCache(SCatalog *pCtg, const char *user, const char *dbFName, AUTH_TYPE type, bool *inCache, + bool *pass) { if (NULL == pCtg->userCache) { ctgDebug("empty user auth cache, user:%s", user); goto _return; } - + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); if (NULL == pUser) { ctgDebug("user not in cache, user:%s", user); @@ -967,7 +930,7 @@ int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFNam ctgDebug("Got user from cache, user:%s", user); CTG_CACHE_STAT_ADD(userHitNum, 1); - + if (pUser->superUser) { *pass = true; return TSDB_CODE_SUCCESS; @@ -979,32 +942,33 @@ int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFNam CTG_UNLOCK(CTG_READ, &pUser->lock); return TSDB_CODE_SUCCESS; } - + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { *pass = true; } - + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { *pass = true; } CTG_UNLOCK(CTG_READ, &pUser->lock); - + return TSDB_CODE_SUCCESS; _return: *inCache = false; CTG_CACHE_STAT_ADD(userMissNum, 1); - + return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnodeImpl(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, char *dbFName, char *tbName, + STableMetaOutput *output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; - char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; - int32_t msgLen = 0; + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); @@ -1016,22 +980,23 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* SRpcMsg rpcMsg = { .msgType = TDMT_MND_TABLE_META, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - + rpcSendRecv(pTrans, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); + if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { SET_META_TYPE_NULL(output->metaType); ctgDebug("stablemeta not exist in mnode, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - - ctgError("error rsp for stablemeta from mnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tbName); + + ctgError("error rsp for stablemeta from mnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, + tbName); CTG_ERR_RET(rpcRsp.code); } @@ -1046,15 +1011,18 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnode(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + STableMetaOutput *output) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); return ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, dbFName, (char *)pTableName->tname, output); } -int32_t ctgGetTableMetaFromVnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { +int32_t ctgGetTableMetaFromVnodeImpl(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + SVgroupInfo *vgroupInfo, STableMetaOutput *output) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || + NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1063,39 +1031,43 @@ int32_t ctgGetTableMetaFromVnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* ctgDebug("try to get table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; - char *msg = NULL; + SBuildTableMetaInput bInput = { + .vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; + char *msg = NULL; int32_t msgLen = 0; int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen); if (code) { - ctgError("Build vnode tablemeta msg failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tNameGetTableName(pTableName)); + ctgError("Build vnode tablemeta msg failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, + tNameGetTableName(pTableName)); CTG_ERR_RET(code); } SRpcMsg rpcMsg = { .msgType = TDMT_VND_TABLE_META, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); - + if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { SET_META_TYPE_NULL(output->metaType); ctgDebug("tablemeta not exist in vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); return TSDB_CODE_SUCCESS; } - - ctgError("error rsp for table meta from vnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tNameGetTableName(pTableName)); + + ctgError("error rsp for table meta from vnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, + tNameGetTableName(pTableName)); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process vnode tablemeta rsp failed, code:%s, dbFName:%s, tbName:%s", tstrerror(code), dbFName, tNameGetTableName(pTableName)); + ctgError("Process vnode tablemeta rsp failed, code:%s, dbFName:%s, tbName:%s", tstrerror(code), dbFName, + tNameGetTableName(pTableName)); CTG_ERR_RET(code); } @@ -1103,17 +1075,18 @@ int32_t ctgGetTableMetaFromVnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromVnode(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + SVgroupInfo *vgroupInfo, STableMetaOutput *output) { int32_t code = 0; int32_t retryNum = 0; - + while (retryNum < CTG_DEFAULT_MAX_RETRY_TIMES) { code = ctgGetTableMetaFromVnodeImpl(pCtg, pTrans, pMgmtEps, pTableName, vgroupInfo, output); if (code) { if (TSDB_CODE_VND_HASH_MISMATCH == code) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - + code = catalogRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName); if (code != TSDB_CODE_SUCCESS) { break; @@ -1140,17 +1113,17 @@ int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { return TSDB_CODE_SUCCESS; } -int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) { - SHashObj *vgroupHash = NULL; +int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray **pList) { + SHashObj *vgroupHash = NULL; SVgroupInfo *vgInfo = NULL; - SArray *vgList = NULL; - int32_t code = 0; - int32_t vgNum = taosHashGetSize(vgHash); + SArray *vgList = NULL; + int32_t code = 0; + int32_t vgNum = taosHashGetSize(vgHash); vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { ctgError("taosArrayInit failed, num:%d", vgNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } void *pIter = taosHashIterate(vgHash, NULL); @@ -1159,10 +1132,10 @@ int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) { if (NULL == taosArrayPush(vgList, vgInfo)) { ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); - taosHashCancelIterate(vgHash, pIter); + taosHashCancelIterate(vgHash, pIter); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - + pIter = taosHashIterate(vgHash, pIter); vgInfo = NULL; } @@ -1184,9 +1157,9 @@ _return: int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { int32_t code = 0; - + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); - char db[TSDB_DB_FNAME_LEN] = {0}; + char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); if (vgNum <= 0) { @@ -1195,7 +1168,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName } tableNameHashFp fp = NULL; - SVgroupInfo *vgInfo = NULL; + SVgroupInfo *vgInfo = NULL; CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); @@ -1211,13 +1184,14 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName taosHashCancelIterate(dbInfo->vgHash, pIter); break; } - + pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } if (NULL == vgInfo) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash)); + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, + taosHashGetSize(dbInfo->vgHash)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1226,54 +1200,53 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } -int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { - if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { +int32_t ctgStbVersionSearchCompare(const void *key1, const void *key2) { + if (*(uint64_t *)key1 < ((SSTableMetaVersion *)key2)->suid) { return -1; - } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { + } else if (*(uint64_t *)key1 > ((SSTableMetaVersion *)key2)->suid) { return 1; } else { return 0; } } -int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { - if (*(int64_t *)key1 < ((SDbVgVersion*)key2)->dbId) { +int32_t ctgDbVgVersionSearchCompare(const void *key1, const void *key2) { + if (*(int64_t *)key1 < ((SDbVgVersion *)key2)->dbId) { return -1; - } else if (*(int64_t *)key1 > ((SDbVgVersion*)key2)->dbId) { + } else if (*(int64_t *)key1 > ((SDbVgVersion *)key2)->dbId) { return 1; } else { return 0; } } -int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { +int32_t ctgStbVersionSortCompare(const void *key1, const void *key2) { + if (((SSTableMetaVersion *)key1)->suid < ((SSTableMetaVersion *)key2)->suid) { return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + } else if (((SSTableMetaVersion *)key1)->suid > ((SSTableMetaVersion *)key2)->suid) { return 1; } else { return 0; } } -int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) { - if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { +int32_t ctgDbVgVersionSortCompare(const void *key1, const void *key2) { + if (((SDbVgVersion *)key1)->dbId < ((SDbVgVersion *)key2)->dbId) { return -1; - } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { + } else if (((SDbVgVersion *)key1)->dbId > ((SDbVgVersion *)key2)->dbId) { return 1; } else { return 0; } } - int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; mgmt->type = type; size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; - + mgmt->slots = taosMemoryCalloc(1, msgSize); if (NULL == mgmt->slots) { qError("calloc %d failed", (int32_t)msgSize); @@ -1281,34 +1254,34 @@ int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { } qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); - + return TSDB_CODE_SUCCESS; } - int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - + int32_t code = 0; + CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size); if (NULL == slot->meta) { - qError("taosArrayInit %d failed, id:%"PRIx64", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, mgmt->type); + qError("taosArrayInit %d failed, id:%" PRIx64 ", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, + mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } if (NULL == taosArrayPush(slot->meta, meta)) { - qError("taosArrayPush meta to rent failed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("taosArrayPush meta to rent failed, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } slot->needSort = true; - qDebug("add meta to rent, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("add meta to rent, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); _return: @@ -1316,20 +1289,22 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { +int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, + __compar_fn_t searchCompare) { int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; + int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { - qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("empty meta slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (slot->needSort) { - qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, + (int32_t)taosArrayGetSize(slot->meta)); taosArraySort(slot->meta, sortCompare); slot->needSort = false; qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); @@ -1337,20 +1312,22 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); if (NULL == orig) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qError("meta not found in slot, id:%" PRIx64 ", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, + (int32_t)taosArrayGetSize(slot->meta)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } memcpy(orig, meta, size); - qDebug("meta in rent updated, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("meta in rent updated, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); _return: CTG_UNLOCK(CTG_WRITE, &slot->lock); if (code) { - qWarn("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); + qWarn("meta in rent update failed, will try to add it, code:%x, id:%" PRIx64 ", slot idx:%d, type:%d", code, id, + widx, mgmt->type); CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); } @@ -1361,11 +1338,11 @@ int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortComp int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - + int32_t code = 0; + CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { - qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("empty meta slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1377,13 +1354,13 @@ int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortComp int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ); if (idx < 0) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("meta not found in slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } taosArrayRemove(slot->meta, idx); - qDebug("meta in rent removed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("meta in rent removed, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); _return: @@ -1392,7 +1369,6 @@ _return: CTG_RET(code); } - int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1); if (ridx >= mgmt->slotNum) { @@ -1401,8 +1377,8 @@ int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_ } SCtgRentSlot *slot = &mgmt->slots[ridx]; - int32_t code = 0; - + int32_t code = 0; + CTG_LOCK(CTG_READ, &slot->lock); if (NULL == slot->meta) { qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type); @@ -1468,13 +1444,15 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { SCtgDBCache newDBCache = {0}; newDBCache.dbId = dbId; - newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == newDBCache.tbCache.metaCache) { ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); if (NULL == newDBCache.tbCache.stbCache) { ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); @@ -1486,21 +1464,21 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ctgDebug("db already in cache, dbFName:%s", dbFName); goto _return; } - + ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_ADD(dbNum, 1); - + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + ctgDebug("db added to cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbId); CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%" PRIx64, dbFName, vgVersion.vgVersion, dbId); return TSDB_CODE_SUCCESS; @@ -1511,8 +1489,7 @@ _return: CTG_RET(code); } - -void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { +void ctgRemoveStbRent(SCatalog *pCtg, SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { void *pIter = taosHashIterate(cache->stbCache, NULL); @@ -1520,21 +1497,21 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { uint64_t *suid = NULL; suid = taosHashGetKey(pIter, NULL); - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { - ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); + if (TSDB_CODE_SUCCESS == + ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + ctgDebug("stb removed from rent, suid:%" PRIx64, *suid); } - + pIter = taosHashIterate(cache->stbCache, pIter); } } CTG_UNLOCK(CTG_WRITE, &cache->stbLock); } - -int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { +int32_t ctgRemoveDB(SCatalog *pCtg, SCtgDBCache *dbCache, const char *dbFName) { uint64_t dbId = dbCache->dbId; - - ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + ctgInfo("start to remove db from cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbCache->dbId); atomic_store_8(&dbCache->deleted, 1); @@ -1543,8 +1520,8 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { ctgFreeDbCache(dbCache); CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + ctgDebug("db removed from rent, dbFName:%s, dbId:%" PRIx64, dbFName, dbCache->dbId); if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); @@ -1553,19 +1530,18 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { CTG_CACHE_STAT_SUB(dbNum, 1); - ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - + ctgInfo("db removed from cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbId); + return TSDB_CODE_SUCCESS; } - -int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { - int32_t code = 0; +int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { + int32_t code = 0; SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, dbFName, &dbCache); - + if (dbCache) { - // TODO OPEN IT + // TODO OPEN IT #if 0 if (dbCache->dbId == dbId) { *pCache = dbCache; @@ -1582,7 +1558,7 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt *pCache = dbCache; return TSDB_CODE_SUCCESS; } - + if (dbCache->dbId == dbId) { *pCache = dbCache; return TSDB_CODE_SUCCESS; @@ -1590,7 +1566,7 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt #endif CTG_ERR_RET(ctgRemoveDB(pCtg, dbCache, dbFName)); } - + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); ctgGetDBCache(pCtg, dbFName, &dbCache); @@ -1600,46 +1576,47 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } - -int32_t ctgUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { - int32_t code = 0; - SDBVgInfo* dbInfo = *pDbInfo; +int32_t ctgUpdateDBVgInfo(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SDBVgInfo **pDbInfo) { + int32_t code = 0; + SDBVgInfo *dbInfo = *pDbInfo; if (NULL == dbInfo->vgHash) { return TSDB_CODE_SUCCESS; } - + if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", - dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash, + dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - bool newAdded = false; + bool newAdded = false; SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; SCtgDBCache *dbCache = NULL; CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, dbFName, dbId); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } SDBVgInfo *vgInfo = NULL; CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - + if (dbCache->vgInfo) { if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) { - ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, + dbCache->vgInfo->vgVersion); ctgWReleaseVgInfo(dbCache); - + return TSDB_CODE_SUCCESS; } if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) { - ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); + ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, + dbInfo->numOfTable); ctgWReleaseVgInfo(dbCache); - + return TSDB_CODE_SUCCESS; } @@ -1650,53 +1627,55 @@ int32_t ctgUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SD *pDbInfo = NULL; - ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); ctgWReleaseVgInfo(dbCache); dbCache = NULL; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), + ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + CTG_RET(code); } - -int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { +int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, + STableMeta *meta, int32_t metaSize) { SCtgTbMetaCache *tbCache = &dbCache->tbCache; CTG_LOCK(CTG_READ, &tbCache->metaLock); if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("db is dropping, dbId:%" PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } - int8_t origType = 0; - uint64_t origSuid = 0; - bool isStb = meta->tableType == TSDB_SUPER_TABLE; + int8_t origType = 0; + uint64_t origSuid = 0; + bool isStb = meta->tableType == TSDB_SUPER_TABLE; STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); if (orig) { origType = orig->tableType; - if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && + orig->tversion >= meta->tversion) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; } - + if (origType == TSDB_SUPER_TABLE) { if ((!isStb) || orig->suid != meta->suid) { CTG_LOCK(CTG_WRITE, &tbCache->stbLock); if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%" PRIx64, dbFName, tbName, orig->suid); } else { CTG_CACHE_STAT_SUB(stblNum, 1); } CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%" PRIx64, dbFName, tbName, orig->suid); + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); } @@ -1707,13 +1686,13 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui if (isStb) { CTG_LOCK(CTG_WRITE, &tbCache->stbLock); } - + if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { if (isStb) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); } - - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1722,35 +1701,38 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_CACHE_STAT_ADD(tblNum, 1); } - ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64, dbFName, tbName, meta->tableType, meta->suid); + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64, dbFName, tbName, meta->tableType, + meta->suid); ctgdShowTableMeta(pCtg, tbName, meta); if (!isStb) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; } STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("taosHashPut stable to stable cache failed, suid:%" PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_ADD(stblNum, 1); - + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64 ",ma:%p", dbFName, tbName, meta->tableType, meta->suid, tbMeta); + ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64 ",ma:%p", dbFName, tbName, + meta->tableType, meta->suid, tbMeta); - SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; + SSTableMetaVersion metaRent = { + .dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; strcpy(metaRent.dbFName, dbFName); strcpy(metaRent.stbName, tbName); CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); - + return TSDB_CODE_SUCCESS; } @@ -1772,7 +1754,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { } int32_t *vgId = NULL; - void *pIter = taosHashIterate(src->vgHash, NULL); + void *pIter = taosHashIterate(src->vgHash, NULL); while (pIter) { vgId = taosHashGetKey(pIter, NULL); @@ -1783,18 +1765,16 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { taosMemoryFreeClear(*dst); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - + pIter = taosHashIterate(src->vgHash, pIter); } - return TSDB_CODE_SUCCESS; } - - -int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { - bool inCache = false; +int32_t ctgGetDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SCtgDBCache **dbCache, + SDBVgInfo **pInfo) { + bool inCache = false; int32_t code = 0; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); @@ -1803,7 +1783,7 @@ int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const return TSDB_CODE_SUCCESS; } - SUseDbOutput DbOut = {0}; + SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); @@ -1829,18 +1809,18 @@ _return: taosMemoryFreeClear(*pInfo); *pInfo = DbOut.dbVgroup; - + CTG_RET(code); } -int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName) { - bool inCache = false; - int32_t code = 0; - SCtgDBCache* dbCache = NULL; +int32_t ctgRefreshDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName) { + bool inCache = false; + int32_t code = 0; + SCtgDBCache *dbCache = NULL; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); - SUseDbOutput DbOut = {0}; + SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); @@ -1850,7 +1830,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, c ctgReleaseVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } - + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; input.numOfTable = 0; @@ -1869,8 +1849,6 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, c return TSDB_CODE_SUCCESS; } - - int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { *pOutput = taosMemoryMalloc(sizeof(STableMetaOutput)); if (NULL == *pOutput) { @@ -1895,15 +1873,14 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) return TSDB_CODE_SUCCESS; } - - -int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t flag, STableMetaOutput **pOutput, bool syncReq) { +int32_t ctgRefreshTblMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, int32_t flag, + STableMetaOutput **pOutput, bool syncReq) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } SVgroupInfo vgroupInfo = {0}; - int32_t code = 0; + int32_t code = 0; if (!CTG_FLAG_IS_SYS_DB(flag)) { CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); @@ -1919,7 +1896,8 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (CTG_FLAG_IS_SYS_DB(flag)) { ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(pTableName)); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, (char *)pTableName->tname, output)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, + (char *)pTableName->tname, output)); } else if (CTG_FLAG_IS_STB(flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); @@ -1939,7 +1917,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pTableName)); taosMemoryFreeClear(output->tbMeta); - + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; @@ -1953,14 +1931,14 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (CTG_IS_META_NULL(moutput.metaType)) { SET_META_TYPE_NULL(output->metaType); } - + taosMemoryFreeClear(output->tbMeta); output->tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { taosMemoryFreeClear(output->tbMeta); - - SET_META_TYPE_CTABLE(output->metaType); + + SET_META_TYPE_CTABLE(output->metaType); } } } @@ -1972,9 +1950,11 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } if (CTG_IS_META_TABLE(output->metaType)) { - ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, output->tbMeta->tableType); + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, + output->tbMeta->tableType); } else { - ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, + output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); } if (pOutput) { @@ -1989,19 +1969,20 @@ _return: taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output); - + CTG_RET(code); } -int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t flag) { +int32_t ctgGetTableMeta(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, + STableMeta **pTableMeta, int32_t flag) { if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - bool inCache = false; - int32_t code = 0; - uint64_t dbId = 0; - uint64_t suid = 0; + bool inCache = false; + int32_t code = 0; + uint64_t dbId = 0; + uint64_t suid = 0; STableMetaOutput *output = NULL; if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { @@ -2013,7 +1994,8 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons int32_t tbType = 0; if (inCache) { - if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_SYS_DB(flag)))) { + if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && + ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_SYS_DB(flag)))) { goto _return; } @@ -2027,7 +2009,6 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_FLAG_SET_STB(flag, tbType); } - while (true) { CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output, false)); @@ -2038,7 +2019,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons if (CTG_IS_META_BOTH(output->metaType)) { memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); - + *pTableMeta = output->tbMeta; goto _return; } @@ -2055,11 +2036,11 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons strcpy(stbName.tname, output->tbName); taosMemoryFreeClear(output->tbMeta); - + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL)); if (!inCache) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); - + continue; } @@ -2095,12 +2076,13 @@ _return: CTG_RET(code); } -int32_t ctgChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { - bool inCache = false; +int32_t ctgChkAuth(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, const char *dbFName, + AUTH_TYPE type, bool *pass) { + bool inCache = false; int32_t code = 0; - + *pass = false; - + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); if (inCache) { @@ -2109,7 +2091,7 @@ int32_t ctgChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const cha SGetUserAuthRsp authRsp = {0}; CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp)); - + if (authRsp.superAuth) { *pass = true; goto _return; @@ -2135,53 +2117,52 @@ _return: return TSDB_CODE_SUCCESS; } - int32_t ctgActUpdateVg(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateVgMsg *msg = action->data; - + CTG_ERR_JRET(ctgUpdateDBVgInfo(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); _return: ctgFreeVgInfo(msg->dbInfo); taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgActRemoveDB(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgRemoveDBMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); if (NULL == dbCache) { goto _return; } - + if (dbCache->dbId != msg->dbId) { - ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); + ctgInfo("dbId already updated, dbFName:%s, dbId:%" PRIx64 ", targetId:%" PRIx64, msg->dbFName, dbCache->dbId, + msg->dbId); goto _return; } - + CTG_ERR_JRET(ctgRemoveDB(pCtg, dbCache, msg->dbFName)); _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } - int32_t ctgActUpdateTbl(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateTblMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; - STableMetaOutput* output = msg->output; - SCtgDBCache *dbCache = NULL; + SCatalog *pCtg = msg->pCtg; + STableMetaOutput *output = msg->output; + SCtgDBCache *dbCache = NULL; if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); @@ -2191,22 +2172,24 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - + } + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, output->dbFName, output->dbId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { int32_t metaSize = CTG_META_SIZE(output->tbMeta); - - CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + + CTG_ERR_JRET( + ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); } if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, + (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); } _return: @@ -2215,17 +2198,16 @@ _return: taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output); } - + taosMemoryFreeClear(msg); - + CTG_RET(code); } - int32_t ctgActRemoveStb(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgRemoveStbMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, msg->dbFName, &dbCache); @@ -2234,44 +2216,46 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { } if (msg->dbId && (dbCache->dbId != msg->dbId)) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); + ctgDebug("dbId already modified, dbFName:%s, current:%" PRIx64 ", dbId:%" PRIx64 ", stb:%s, suid:%" PRIx64, + msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); return TSDB_CODE_SUCCESS; } - + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { - ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, + msg->suid); } else { CTG_CACHE_STAT_SUB(stblNum, 1); } CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { - ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_SUB(tblNum, 1); } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - - ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + + ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); - - ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - + + ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); + _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgActRemoveTbl(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgRemoveTblMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, msg->dbFName, &dbCache); @@ -2280,7 +2264,8 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); + ctgDebug("dbId already modified, dbFName:%s, current:%" PRIx64 ", dbId:%" PRIx64 ", tbName:%s", msg->dbFName, + dbCache->dbId, msg->dbId, msg->tbName); return TSDB_CODE_SUCCESS; } @@ -2304,18 +2289,19 @@ _return: } int32_t ctgActUpdateUser(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateUserMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; if (NULL == pCtg->userCache) { - pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), + false, HASH_ENTRY_LOCK); if (NULL == pCtg->userCache) { ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } - + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); if (NULL == pUser) { SCtgUserAuth userAuth = {0}; @@ -2326,7 +2312,7 @@ int32_t ctgActUpdateUser(SCtgMetaAction *action) { userAuth.readDbs = msg->userAuth.readDbs; userAuth.writeDbs = msg->userAuth.writeDbs; - if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { + if (taosHashPut(pCtg->userCache, msg->userAuth.user, sizeof(msg->userAuth.user), &userAuth, sizeof(userAuth))) { ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -2356,30 +2342,28 @@ int32_t ctgActUpdateUser(SCtgMetaAction *action) { _return: - taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); - + taosMemoryFreeClear(msg); - + CTG_RET(code); } - -void* ctgUpdateThreadFunc(void* param) { +void *ctgUpdateThreadFunc(void *param) { setThreadName("catalog"); qInfo("catalog update thread started"); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); - + while (true) { if (tsem_wait(&gCtgMgmt.queue.reqSem)) { qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } - - if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + + if (atomic_load_8((int8_t *)&gCtgMgmt.exit)) { tsem_post(&gCtgMgmt.queue.rspSem); break; } @@ -2389,7 +2373,7 @@ void* ctgUpdateThreadFunc(void* param) { SCatalog *pCtg = ((SCtgUpdateMsgHeader *)action->data)->pCtg; ctgDebug("process [%s] action", gCtgAction[action->act].name); - + (*gCtgAction[action->act].func)(action); gCtgMgmt.queue.seqDone = action->seqId; @@ -2398,7 +2382,7 @@ void* ctgUpdateThreadFunc(void* param) { tsem_post(&gCtgMgmt.queue.rspSem); } - CTG_RUNTIME_STAT_ADD(qDoneNum, 1); + CTG_RUNTIME_STAT_ADD(qDoneNum, 1); ctgdShowClusterCache(pCtg); } @@ -2406,11 +2390,10 @@ void* ctgUpdateThreadFunc(void* param) { CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); qInfo("catalog update thread stopped"); - + return NULL; } - int32_t ctgStartUpdateThread() { TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); @@ -2420,27 +2403,28 @@ int32_t ctgStartUpdateThread() { terrno = TAOS_SYSTEM_ERROR(errno); CTG_ERR_RET(terrno); } - + taosThreadAttrDestroy(&thAttr); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { - STableMeta *tbMeta = NULL; - int32_t code = 0; - SVgroupInfo vgroupInfo = {0}; - SCtgDBCache* dbCache = NULL; - SArray *vgList = NULL; - SDBVgInfo *vgInfo = NULL; +int32_t ctgGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, + SArray **pVgList) { + STableMeta *tbMeta = NULL; + int32_t code = 0; + SVgroupInfo vgroupInfo = {0}; + SCtgDBCache *dbCache = NULL; + SArray *vgList = NULL; + SDBVgInfo *vgInfo = NULL; *pVgList = NULL; - + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, &tbMeta, CTG_FLAG_UNKNOWN_STB)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - SHashObj *vgHash = NULL; + SHashObj *vgHash = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, &dbCache, &vgInfo)); if (dbCache) { @@ -2455,7 +2439,7 @@ int32_t ctgGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps // USE HASH METHOD INSTEAD OF VGID IN TBMETA ctgError("invalid method to get none stb vgInfo, tbType:%d", tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); - + #if 0 int32_t vgId = tbMeta->vgId; if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) { @@ -2476,7 +2460,7 @@ int32_t ctgGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps *pVgList = vgList; vgList = NULL; -#endif +#endif } _return: @@ -2507,7 +2491,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - atomic_store_8((int8_t*)&gCtgMgmt.exit, false); + atomic_store_8((int8_t *)&gCtgMgmt.exit, false); if (cfg) { memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); @@ -2534,7 +2518,8 @@ int32_t catalogInit(SCatalogCfg *cfg) { gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } - gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), + false, HASH_ENTRY_LOCK); if (NULL == gCtgMgmt.pCluster) { qError("taosHashInit %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -2544,7 +2529,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); } - + if (tsem_init(&gCtgMgmt.queue.rspSem, 0, 0)) { qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); @@ -2559,30 +2544,31 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(ctgStartUpdateThread()); - qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, + gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } -int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { +int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) { if (NULL == catalogHandle) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } if (NULL == gCtgMgmt.pCluster) { - qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId); + qError("catalog cluster cache are not ready, clusterId:%" PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } - int32_t code = 0; + int32_t code = 0; SCatalog *clusterCtg = NULL; while (true) { - SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); + SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char *)&clusterId, sizeof(clusterId)); if (ctg && (*ctg)) { *catalogHandle = *ctg; - qDebug("got catalog handle from cache, clusterId:%"PRIx64", CTG:%p", clusterId, *ctg); + qDebug("got catalog handle from cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, *ctg); return TSDB_CODE_SUCCESS; } @@ -2597,7 +2583,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB)); CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); - clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), + false, HASH_ENTRY_LOCK); if (NULL == clusterCtg->dbCache) { qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); @@ -2609,12 +2596,12 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { ctgFreeHandle(clusterCtg); continue; } - - qError("taosHashPut CTG to cache failed, clusterId:%"PRIx64, clusterId); + + qError("taosHashPut CTG to cache failed, clusterId:%" PRIx64, clusterId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("add CTG to cache, clusterId:%"PRIx64", CTG:%p", clusterId, clusterCtg); + qDebug("add CTG to cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, clusterCtg); break; } @@ -2622,36 +2609,36 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { *catalogHandle = clusterCtg; CTG_CACHE_STAT_ADD(clusterNum, 1); - + return TSDB_CODE_SUCCESS; _return: ctgFreeHandle(clusterCtg); - + CTG_RET(code); } -void catalogFreeHandle(SCatalog* pCtg) { +void catalogFreeHandle(SCatalog *pCtg) { if (NULL == pCtg) { return; } if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { - ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCtg->clusterId); + ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%" PRIx64, pCtg->clusterId); return; } CTG_CACHE_STAT_SUB(clusterNum, 1); uint64_t clusterId = pCtg->clusterId; - + ctgFreeHandle(pCtg); - - ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); + + ctgInfo("handle freed, culsterId:%" PRIx64, clusterId); } -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t *tableNum) { +int32_t catalogGetDBVgVersion(SCatalog *pCtg, const char *dbFName, int32_t *version, int64_t *dbId, int32_t *tableNum) { CTG_API_ENTER(); if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { @@ -2659,8 +2646,8 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers } SCtgDBCache *dbCache = NULL; - bool inCache = false; - int32_t code = 0; + bool inCache = false; + int32_t code = 0; CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); if (!inCache) { @@ -2684,18 +2671,19 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { +int32_t catalogGetDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, + SArray **vgroupList) { CTG_API_ENTER(); if (NULL == pCtg || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SCtgDBCache* dbCache = NULL; - int32_t code = 0; - SArray *vgList = NULL; - SHashObj *vgHash = NULL; - SDBVgInfo *vgInfo = NULL; + SCtgDBCache *dbCache = NULL; + int32_t code = 0; + SArray *vgList = NULL; + SHashObj *vgHash = NULL; + SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName, &dbCache, &vgInfo)); if (dbCache) { vgHash = dbCache->vgInfo->vgHash; @@ -2720,15 +2708,14 @@ _return: taosMemoryFreeClear(vgInfo); } - CTG_API_LEAVE(code); + CTG_API_LEAVE(code); } - -int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo* dbInfo) { +int32_t catalogUpdateDBVgInfo(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SDBVgInfo *dbInfo) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { ctgFreeVgInfo(dbInfo); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); @@ -2741,12 +2728,11 @@ _return: CTG_API_LEAVE(code); } - -int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { +int32_t catalogRemoveDB(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2758,21 +2744,19 @@ int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_ERR_JRET(ctgPushRmDBMsgInQueue(pCtg, dbFName, dbId)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); - + _return: CTG_API_LEAVE(code); } -int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet) { - return 0; -} +int32_t catalogUpdateVgEpSet(SCatalog *pCtg, const char *dbFName, int32_t vgId, SEpSet *epSet) { return 0; } -int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { +int32_t catalogRemoveTableMeta(SCatalog *pCtg, const SName *pTableName) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2782,8 +2766,8 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { } STableMeta *tblMeta = NULL; - bool inCache = false; - uint64_t dbId = 0; + bool inCache = false; + uint64_t dbId = 0; CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId)); if (!inCache) { @@ -2793,13 +2777,14 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - + if (TSDB_SUPER_TABLE == tblMeta->tableType) { CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, tblMeta->suid, true)); } else { CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, true)); } + ctgDebug("table meta %s.%s removed", dbFName, pTableName->tname); _return: @@ -2808,12 +2793,11 @@ _return: CTG_API_LEAVE(code); } - -int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid) { +int32_t catalogRemoveStbMeta(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *stbName, uint64_t suid) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName || NULL == stbName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2825,29 +2809,32 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid, true)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); - + _return: CTG_API_LEAVE(code); } -int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, const char *pIndexName, SIndexMeta** pIndexMeta) { +int32_t catalogGetIndexMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + const char *pIndexName, SIndexMeta **pIndexMeta) { return 0; } -int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + STableMeta **pTableMeta) { CTG_API_ENTER(); CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_UNKNOWN_STB)); } -int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetSTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + STableMeta **pTableMeta) { CTG_API_ENTER(); CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_STB)); } -int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { +int32_t catalogUpdateSTableMeta(SCatalog *pCtg, STableMetaRsp *rspMsg) { CTG_API_ENTER(); if (NULL == pCtg || NULL == rspMsg) { @@ -2859,42 +2846,41 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); } - + int32_t code = 0; strcpy(output->dbFName, rspMsg->dbFName); strcpy(output->tbName, rspMsg->tbName); output->dbId = rspMsg->dbId; - + SET_META_TYPE_TABLE(output->metaType); - + CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta)); CTG_ERR_JRET(ctgPushUpdateTblMsgInQueue(pCtg, output, false)); CTG_API_LEAVE(code); - + _return: taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output); - + CTG_API_LEAVE(code); } - - -int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver, int32_t *tbType, uint64_t *suid, char* stbName) { +int32_t ctgGetTbSverFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tbType, uint64_t *suid, + char *stbName) { *sver = -1; - + if (NULL == pCtg->dbCache) { ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } SCtgDBCache *dbCache = NULL; - char dbFName[TSDB_DB_FNAME_LEN] = {0}; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, dbFName); ctgAcquireDBCache(pCtg, dbFName, &dbCache); @@ -2902,9 +2888,9 @@ int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* ctgDebug("db %s not in cache", pTableName->tname); return TSDB_CODE_SUCCESS; } - + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta* tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); + STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); if (tbMeta) { *tbType = tbMeta->tableType; *suid = tbMeta->suid; @@ -2922,32 +2908,33 @@ int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* if (*tbType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname); - + return TSDB_CODE_SUCCESS; } ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid); - + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid)); if (NULL == stbMeta || NULL == *stbMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("stb not in stbCache, suid:%"PRIx64, *suid); + ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); return TSDB_CODE_SUCCESS; } if ((*stbMeta)->suid != *suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); - ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, *suid, (*stbMeta)->suid); + ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, *suid, + (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } size_t nameLen = 0; - char* name = taosHashGetKey(*stbMeta, &nameLen); - + char *name = taosHashGetKey(*stbMeta, &nameLen); + strncpy(stbName, name, nameLen); stbName[nameLen] = 0; @@ -2958,38 +2945,42 @@ int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* ctgReleaseDBCache(pCtg, dbCache); ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname); - + return TSDB_CODE_SUCCESS; } -int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { +int32_t catalogChkTbMetaVersion(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, SArray *pTables) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTables) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SName name; + SName name; int32_t sver = 0; int32_t tbNum = taosArrayGetSize(pTables); for (int32_t i = 0; i < tbNum; ++i) { STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); - tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (NULL == pTb->tbFName || 0 == pTb->tbFName[0]) { + continue; + } + tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (CTG_IS_SYS_DBNAME(name.dbname)) { continue; } - int32_t tbType = 0; + int32_t tbType = 0; uint64_t suid = 0; - char stbName[TSDB_TABLE_FNAME_LEN]; + char stbName[TSDB_TABLE_FNAME_LEN]; ctgGetTbSverFromCache(pCtg, &name, &sver, &tbType, &suid, stbName); if (sver >= 0 && sver < pTb->sver) { switch (tbType) { case TSDB_CHILD_TABLE: { SName stb = name; strcpy(stb.tname, stbName); - catalogRemoveTableMeta(pCtg, &stb); + catalogRemoveTableMeta(pCtg, &stb); break; } case TSDB_SUPER_TABLE: @@ -3006,8 +2997,7 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - -int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { +int32_t catalogRefreshDBVgInfo(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const char *dbFName) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName) { @@ -3017,23 +3007,28 @@ int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmt CTG_API_LEAVE(ctgRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName)); } -int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { +int32_t catalogRefreshTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + int32_t isSTable) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL, true)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, + CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL, true)); } -int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { +int32_t catalogRefreshGetTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + STableMeta **pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable))); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, + CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable))); } -int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { +int32_t catalogGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, + SArray **pVgList) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { @@ -3051,12 +3046,13 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm code = ctgGetTableDistVgInfo(pCtg, pRpc, pMgmtEps, pTableName, pVgList); if (code) { if (TSDB_CODE_CTG_VG_META_MISMATCH == code) { - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(CTG_FLAG_UNKNOWN_STB), NULL, true)); + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, + CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(CTG_FLAG_UNKNOWN_STB), NULL, true)); char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + tNameGetFullDbName(pTableName, dbFName); CTG_ERR_JRET(ctgRefreshDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName)); - + continue; } } @@ -3069,8 +3065,8 @@ _return: CTG_API_LEAVE(code); } - -int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { +int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + SVgroupInfo *pVgroup) { CTG_API_ENTER(); if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { @@ -3078,9 +3074,9 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pM CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SCtgDBCache* dbCache = NULL; - int32_t code = 0; - char db[TSDB_DB_FNAME_LEN] = {0}; + SCtgDBCache *dbCache = NULL; + int32_t code = 0; + char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); SDBVgInfo *vgInfo = NULL; @@ -3103,8 +3099,8 @@ _return: CTG_API_LEAVE(code); } - -int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { +int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SCatalogReq *pReq, + SMetaData *pRsp) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { @@ -3126,11 +3122,11 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, ctgError("taosArrayInit %d failed", tbNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - + for (int32_t i = 0; i < tbNum; ++i) { - SName *name = taosArrayGet(pReq->pTableName, i); + SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { @@ -3148,7 +3144,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, CTG_API_LEAVE(TSDB_CODE_SUCCESS); -_return: +_return: if (pRsp->pTableMeta) { int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); @@ -3156,19 +3152,19 @@ _return: STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i); taosMemoryFreeClear(pMeta); } - + taosArrayDestroy(pRsp->pTableMeta); pRsp->pTableMeta = NULL; } - + CTG_API_LEAVE(code); } -int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { +int32_t catalogGetQnodeList(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SArray *pQnodeList) { CTG_API_ENTER(); - + int32_t code = 0; - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3179,7 +3175,7 @@ _return: CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) { +int32_t catalogGetExpiredSTables(SCatalog *pCtg, SSTableMetaVersion **stables, uint32_t *num) { CTG_API_ENTER(); if (NULL == pCtg || NULL == stables || NULL == num) { @@ -3189,9 +3185,9 @@ int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, u CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); } -int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) { +int32_t catalogGetExpiredDBs(SCatalog *pCtg, SDbVgVersion **dbs, uint32_t *num) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == dbs || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3199,9 +3195,9 @@ int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } -int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num) { +int32_t catalogGetExpiredUsers(SCatalog *pCtg, SUserAuthVersion **users, uint32_t *num) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == users || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3215,7 +3211,7 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_ } } - uint32_t i = 0; + uint32_t i = 0; SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL); while (pAuth != NULL) { void *key = taosHashGetKey(pAuth, NULL); @@ -3227,10 +3223,9 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_ CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - -int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { +int32_t catalogGetDBCfg(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SDbCfgInfo *pDbCfg) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == dbFName || NULL == pDbCfg) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3238,9 +3233,10 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_API_LEAVE(ctgGetDBCfgFromMnode(pCtg, pRpc, pMgmtEps, dbFName, pDbCfg)); } -int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo) { +int32_t catalogGetIndexInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *indexName, + SIndexInfo *pInfo) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == indexName || NULL == pInfo) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3248,9 +3244,9 @@ int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pRpc, pMgmtEps, indexName, pInfo)); } -int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo) { +int32_t catalogGetUdfInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *funcName, SFuncInfo **pInfo) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3262,32 +3258,33 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, co } CTG_ERR_JRET(ctgGetUdfInfoFromMnode(pCtg, pRpc, pMgmtEps, funcName, pInfo)); - + _return: if (code) { - taosMemoryFreeClear(*pInfo); + taosMemoryFreeClear(*pInfo); } - + CTG_API_LEAVE(code); } -int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { +int32_t catalogChkAuth(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, const char *dbFName, + AUTH_TYPE type, bool *pass) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; CTG_ERR_JRET(ctgChkAuth(pCtg, pRpc, pMgmtEps, user, dbFName, type, pass)); - + _return: CTG_API_LEAVE(code); } -int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { +int32_t catalogUpdateUserAuthInfo(SCatalog *pCtg, SGetUserAuthRsp *pAuth) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pAuth) { @@ -3297,20 +3294,19 @@ int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { CTG_API_LEAVE(ctgPushUpdateUserMsgInQueue(pCtg, pAuth, false)); } - void catalogDestroy(void) { qInfo("start to destroy catalog"); - - if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + + if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t *)&gCtgMgmt.exit)) { return; } - atomic_store_8((int8_t*)&gCtgMgmt.exit, true); + atomic_store_8((int8_t *)&gCtgMgmt.exit, true); if (tsem_post(&gCtgMgmt.queue.reqSem)) { qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } - + if (tsem_post(&gCtgMgmt.queue.rspSem)) { qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } @@ -3318,21 +3314,21 @@ void catalogDestroy(void) { while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { taosUsleep(1); } - + CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); SCatalog *pCtg = NULL; - void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); while (pIter) { pCtg = *(SCatalog **)pIter; if (pCtg) { catalogFreeHandle(pCtg); } - + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); } - + taosHashCleanup(gCtgMgmt.pCluster); gCtgMgmt.pCluster = NULL; @@ -3341,5 +3337,3 @@ void catalogDestroy(void) { qInfo("catalog destroyed"); } - - diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index edc21626f55e70cd0b6fb473451bf276f9589cc6..63c398618f38446260124978a803b2a63c6f0688 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -42,11 +42,8 @@ #define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) typedef struct SGroupResInfo { - int32_t totalGroup; - int32_t currentGroup; int32_t index; - SArray* pRows; // SArray - bool ordered; + SArray* pRows; // SArray int32_t position; } SGroupResInfo; @@ -80,18 +77,12 @@ typedef struct SResultRowInfo { SResultRowPosition cur; } SResultRowInfo; -struct STaskAttr; -struct STaskRuntimeEnv; -struct SUdfInfo; struct SqlFunctionCtx; -int32_t getOutputInterResultBufSize(struct STaskAttr* pQueryAttr); - size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -void resetResultRowInfo(struct STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); void closeAllResultRows(SResultRowInfo* pResultRowInfo); @@ -99,9 +90,7 @@ void initResultRow(SResultRow *pResultRow); void closeResultRow(SResultRow* pResultRow); bool isResultRowClosed(SResultRow* pResultRow); -struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); - -int32_t getRowNumForMultioutput(struct STaskAttr* pQueryAttr, bool topBottomQuery, bool stable); +struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, const int32_t* offset); static FORCE_INLINE SResultRow *getResultRow(SDiskbasedBuf* pBuf, SResultRowInfo *pResultRowInfo, int32_t slot) { ASSERT(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); @@ -118,39 +107,13 @@ static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo return pRow; } -static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFilePage* page, int32_t rowOffset, - int32_t offset) { - assert(rowOffset >= 0 && pQueryAttr != NULL); - - // int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); - return ((char *)page->data); - -} - -static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffset, int32_t offset) { - assert(rowOffset >= 0); - - int32_t numOfRows = 1;//(int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); - return (char*) page + rowOffset + offset * numOfRows; -} - -typedef struct { - SArray* pResult; // SArray - int32_t colId; -} SStddevInterResult; - void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); -bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo); -bool hasRemainData(SGroupResInfo* pGroupResInfo); +bool hashRemainDataInGroupInfo(SGroupResInfo* pGroupResInfo); bool incNextGroup(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); -int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct STaskRuntimeEnv *pRuntimeEnv, int32_t* offset); - -//int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); - #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 78f3266d745046582679222f18137e6ced87fb6c..0dacbba8e590b010405571727775cdcda983629c 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -666,7 +666,6 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); -void finalizeMultiTupleQueryResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 14cff488ecd183b0567fa20709657d44c84a1e47..5a02547f58aa4cf73c5297dda771ba0900bce141 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -114,12 +114,6 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) { assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); for (int32_t i = 0; i < pResultRowInfo->size; ++i) { -// SResultRow* pRow = pResultRowInfo->pResult[i]; -// if (pRow->closed) { -// continue; -// } - -// pRow->closed = true; } } @@ -144,11 +138,11 @@ void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { for (int32_t i = 0; i < pRuntimeEnv->pQueryAttr->numOfOutput; ++i) { struct SResultRowEntryInfo *pEntryInfo = NULL;//pResultRow->pEntryInfo[i]; - int16_t size = pRuntimeEnv->pQueryAttr->pExpr1[i].base.resSchema.bytes; - char * s = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResultRow->offset, offset); - memset(s, 0, size); +// int16_t size = pRuntimeEnv->pQueryAttr->pExpr1[i].base.resSchema.bytes; +// char * s = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResultRow->offset, offset); +// memset(s, 0, size); - offset += size; +// offset += size; cleanupResultRowEntry(pEntryInfo); } } @@ -161,7 +155,7 @@ void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { } // TODO refactor: use macro -SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { +SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, const int32_t* offset) { assert(index >= 0 && offset != NULL); return (SResultRowEntryInfo*)((char*) pRow->pEntryInfo + offset[index]); } @@ -247,7 +241,7 @@ void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL ASSERT(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } -bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo) { +bool hashRemainDataInGroupInfo(SGroupResInfo* pGroupResInfo) { if (pGroupResInfo->pRows == NULL) { return false; } @@ -255,18 +249,6 @@ bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo) { return pGroupResInfo->index < taosArrayGetSize(pGroupResInfo->pRows); } -bool hasRemainData(SGroupResInfo* pGroupResInfo) { - if (hasRemainDataInCurrentGroup(pGroupResInfo)) { - return true; - } - - return pGroupResInfo->currentGroup < pGroupResInfo->totalGroup; -} - -bool incNextGroup(SGroupResInfo* pGroupResInfo) { - return (++pGroupResInfo->currentGroup) < pGroupResInfo->totalGroup; -} - int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); if (pGroupResInfo->pRows == 0) { @@ -387,11 +369,6 @@ void orderTheResultRows(STaskRuntimeEnv* pRuntimeEnv) { } static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, uint64_t groupId, int32_t* rowCellInfoOffset) { - if (!pGroupResInfo->ordered) { - orderTheResultRows(pRuntimeEnv); - pGroupResInfo->ordered = true; - } - if (pGroupResInfo->pRows == NULL) { pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES); } @@ -403,6 +380,7 @@ static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupRe break; } + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, &pResultRowCell->pos, rowCellInfoOffset); if (num <= 0) { continue; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ccffe775141fe57323707de44c73057edd0e90e5..b1b0e69084c7a35ef9bdabacd40515f46b8a7c6f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -156,7 +156,7 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, void operatorDummyCloseFn(void* param, int32_t numOfCols) {} static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs); + const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); @@ -182,10 +182,11 @@ static int compareRowData(const void* a, const void* b, const void* userData) { SFilePage* page2 = getBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId); int16_t offset = supporter->dataOffset; - char* in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset); - char* in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset); + return 0; +// char* in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset); +// char* in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset); - return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; +// return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; } // setup the output buffer for each operator @@ -333,6 +334,8 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, return NULL; } + setBufPageDirty(pData, true); + // set the number of rows in current disk page SResultRow* pResultRow = (SResultRow*)((char*)pData + pData->num); pResultRow->pageId = pageId; @@ -364,12 +367,14 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (isIntervalQuery) { if (masterscan && p1 != NULL) { // the *p1 may be NULL in case of sliding+offset exists. pResult = getResultRowByPos(pResultBuf, p1); + ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); } } else { // In case of group by column query, the required SResultRow object must be existInCurrentResusltRowInfo in the // pResultRowInfo object. if (p1 != NULL) { pResult = getResultRowByPos(pResultBuf, p1); + ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); } } @@ -1417,21 +1422,6 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { // } //} -static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t MIN_ROWS_PER_PAGE = 4; - - *rowsize = (int32_t)(pQueryAttr->resultRowSize * - getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); - int32_t overhead = sizeof(SFilePage); - - // one page contains at least two rows - *ps = DEFAULT_INTERN_BUF_PAGE_SIZE; - while (((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) { - *ps = ((*ps) << 1u); - } -} - // static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, // SqlFunctionCtx *pCtx, int32_t numOfRows) { // STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1708,36 +1698,6 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc return TSDB_CODE_SUCCESS; } -void copyToSDataBlock(SSDataBlock* pBlock, int32_t* offset, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pResBuf) { - pBlock->info.rows = 0; - - int32_t code = TSDB_CODE_SUCCESS; - while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { - // all results in current group have been returned to client, try next group - if ((pGroupResInfo->pRows == NULL) || taosArrayGetSize(pGroupResInfo->pRows) == 0) { - assert(pGroupResInfo->index == 0); - // if ((code = mergeIntoGroupResult(&pGroupResInfo, pRuntimeEnv, offset)) != TSDB_CODE_SUCCESS) { - return; - // } - } - - // doCopyToSDataBlock(pResBuf, pGroupResInfo, TSDB_ORDER_ASC, pBlock, ); - - // current data are all dumped to result buffer, clear it - if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { - cleanupGroupResInfo(pGroupResInfo); - if (!incNextGroup(pGroupResInfo)) { - break; - } - } - - // enough results in data buffer, return - // if (pBlock->info.rows >= threshold) { - // break; - // } - } -} - static void updateTableQueryInfoForReverseScan(STableQueryInfo* pTableQueryInfo) { if (pTableQueryInfo == NULL) { return; @@ -1884,35 +1844,6 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { } } -// todo merged with the build group result. -void finalizeMultiTupleQueryResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, - int32_t* rowCellInfoOffset) { - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRowPosition* pPos = &pResultRowInfo->pPosition[i]; - - SFilePage* bufPage = getBufPage(pBuf, pPos->pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset); - - // TODO ignore the close status anyway. - // if (!isResultRowClosed(pRow)) { - // continue; - // } - - for (int32_t j = 0; j < numOfOutput; ++j) { - struct SResultRowEntryInfo* pResInfo = getResultCell(pRow, j, rowCellInfoOffset); - if (!isRowEntryInitialized(pResInfo)) { - continue; - } - - if (pRow->numOfRows < pResInfo->numOfRes) { - pRow->numOfRows = pResInfo->numOfRes; - } - } - - releaseBufPage(pBuf, bufPage); - } -} - STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win) { STableQueryInfo* pTableQueryInfo = buf; pTableQueryInfo->lastKey = win.skey; @@ -1972,7 +1903,7 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchIn filterFreeInfo(filter); extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); - blockDataUpdateTsWindow(pBlock); + blockDataUpdateTsWindow(pBlock, 0); } void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) { @@ -2062,20 +1993,34 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p pAggInfo->groupId = groupId; } +static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_t* rowCellOffset) { + for (int32_t j = 0; j < numOfExprs; ++j) { + struct SResultRowEntryInfo* pResInfo = getResultCell(pRow, j, rowCellOffset); + if (!isRowEntryInitialized(pResInfo)) { + continue; + } + + if (pRow->numOfRows < pResInfo->numOfRes) { + pRow->numOfRows = pResInfo->numOfRes; + } + } +} + int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { + const int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t start = pGroupResInfo->index; - // qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_TASKID(pRuntimeEnv)); - for (int32_t i = start; i < numOfRows; i += 1) { SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); + + doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; + releaseBufPage(pBuf, page); continue; } @@ -2084,11 +2029,13 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI } else { // current value belongs to different group, it can't be packed into one datablock if (pBlock->info.groupId != pPos->groupId) { + releaseBufPage(pBuf, page); break; } } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + releaseBufPage(pBuf, page); break; } @@ -2125,13 +2072,11 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI } qDebug("%s result generated, rows:%d, groupId:%"PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.groupId); - blockDataUpdateTsWindow(pBlock); + blockDataUpdateTsWindow(pBlock, 0); return 0; } void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { - assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); - SExprInfo* pExprInfo = pOperator->pExpr; int32_t numOfExprs = pOperator->numOfExprs; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2141,7 +2086,7 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG SqlFunctionCtx* pCtx = pbInfo->pCtx; blockDataCleanup(pBlock); - if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { + if (!hashRemainDataInGroupInfo(pGroupResInfo)) { return; } @@ -2852,7 +2797,9 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI } pRes->info.rows = numOfRows; - blockDataUpdateTsWindow(pRes); + + // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. + blockDataUpdateTsWindow(pRes, 0); int64_t el = taosGetTimestampUs() - startTs; @@ -3627,9 +3574,6 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { } closeAllResultRows(&pAggInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pOperator->numOfExprs, pAggInfo->aggSup.pResultBuf, &pAggInfo->binfo.resultRowInfo, - pAggInfo->binfo.rowCellInfoOffset); - initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; @@ -3651,7 +3595,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { + if (pInfo->pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -4115,7 +4059,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n defaultBufsz = defaultPgsz * 4; } - int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, "/tmp/"); + int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, TD_TMP_DIR_PATH); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index b1e19213dd6ef57a6a687348a6443fc0b60704c1..484d439dea47e83e99edcbe02dd543399f17f9de 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -269,7 +269,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + if (pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } return (pRes->info.rows == 0)? NULL:pRes; @@ -303,9 +303,6 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pInfo->binfo.resultRowInfo); - - finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, - pInfo->binfo.rowCellInfoOffset); // if (!stableQuery) { // finalize include the update of result rows // finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfExprs); // } else { @@ -320,7 +317,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pRes, NULL); - bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); + bool hasRemain = hashRemainDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { pOperator->status = OP_EXEC_DONE; break; @@ -556,7 +553,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->pageIndex += 1; - blockDataUpdateTsWindow(pInfo->binfo.pRes); + blockDataUpdateTsWindow(pInfo->binfo.pRes, 0); pInfo->binfo.pRes->info.groupId = pGroupInfo->groupId; return pInfo->binfo.pRes; } @@ -617,7 +614,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - int32_t code = createDiskbasedBuf(&pInfo->pBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/"); + int32_t code = createDiskbasedBuf(&pInfo->pBuf, 4096, 4096 * 256, pTaskInfo->id.str, TD_TMP_DIR_PATH); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 2c204e93563367b820eabc27321631935beb2489..123c77ce9b8656654cfef0b02fcf37e1e04a229e 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -19,38 +19,11 @@ #include "nodes.h" #include "tdatablock.h" -typedef struct SIFCtx { - int32_t code; - SHashObj *pRes; /* element is SScalarParam */ - bool noExec; // true: just iterate condition tree, and add hint to executor plan - // SIdxFltStatus st; -} SIFCtx; - -#define SIF_ERR_RET(c) \ - do { \ - int32_t _code = c; \ - if (_code != TSDB_CODE_SUCCESS) { \ - terrno = _code; \ - return _code; \ - } \ - } while (0) -#define SIF_RET(c) \ - do { \ - int32_t _code = c; \ - if (_code != TSDB_CODE_SUCCESS) { \ - terrno = _code; \ - } \ - return _code; \ - } while (0) -#define SIF_ERR_JRET(c) \ - do { \ - code = c; \ - if (code != TSDB_CODE_SUCCESS) { \ - terrno = code; \ - goto _return; \ - } \ - } while (0) - +// clang-format off +#define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define SIF_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define SIF_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) +// clang-format on typedef struct SIFParam { SHashObj *pFilter; @@ -65,6 +38,13 @@ typedef struct SIFParam { char colName[TSDB_COL_NAME_LEN]; } SIFParam; +typedef struct SIFCtx { + int32_t code; + SHashObj *pRes; /* element is SIFParam */ + bool noExec; // true: just iterate condition tree, and add hint to executor plan + // SIdxFltStatus st; +} SIFCtx; + static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { if (src == OP_TYPE_GREATER_THAN) { *dst = QUERY_GREATER_THAN; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8b6ab96b6df3471f88f91d8d277eb1c127112a29..d0eb0ae8dad4a182413af4818bd0ece474e9d238 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -689,7 +689,7 @@ static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool inverti } pDataBlock->info.rows = size; pDataBlock->info.type = STREAM_REPROCESS; - blockDataUpdateTsWindow(pDataBlock); + blockDataUpdateTsWindow(pDataBlock, 0); taosArrayClear(pInfo->tsArray); return pDataBlock; } @@ -899,7 +899,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } rows = pBlockInfo->rows; doFilter(pInfo->pCondition, pInfo->pRes, NULL); - blockDataUpdateTsWindow(pInfo->pRes); + blockDataUpdateTsWindow(pInfo->pRes, 0); break; } @@ -972,9 +972,10 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR } pInfo->primaryTsIndex = 0; // TODO(liuyao) get it from physical plan - pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, 10000); // TODO(liuyao) get watermark from physical plan - if (pInfo->pUpdateInfo == NULL) { - goto _error; + if (pSTInfo->interval.interval > 0) { + pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, 10000); // TODO(liuyao) get watermark from physical plan + } else { + pInfo->pUpdateInfo = NULL; } pInfo->readHandle = *pHandle; @@ -989,7 +990,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR size_t childKeyBufSize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY); initCatchSupporter(&pInfo->childAggSup, 1024, childKeyBufSize, - "StreamFinalInterval", "/tmp/"); // TODO(liuyao) get row size from phy plan + "StreamFinalInterval", TD_TMP_DIR_PATH); // TODO(liuyao) get row size from phy plan pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 3be131e550bbe9e3c8f025ef90dee857421bf5f9..588c3e90e7757feb1ca7dcd1950c14f84b38c577 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -621,7 +621,7 @@ static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowInd } } -static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, +static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, uint64_t tableGroupId) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; @@ -639,13 +639,17 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // int32_t prevIndex = pResultRowInfo->curPos; TSKEY* tsCols = NULL; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + if (pBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; + + if (tsCols != NULL) { + blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); + } } int32_t startPos = 0; - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols, pBlock->info.rows, ascScan); STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); @@ -670,7 +674,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe int32_t forwardStep = 0; TSKEY ekey = ascScan? win.ekey:win.skey; forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); ASSERT(forwardStep > 0); // prev time window not interpolation yet. @@ -686,7 +690,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } STimeWindow w = pRes->win; - ret = setTimeWindowOutputBuf(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, tableGroupId, + ret = setTimeWindowOutputBuf(pResultRowInfo, pBlock->info.uid, &w, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { @@ -694,17 +698,17 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pSDataBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, + doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); - doApplyFunctions(pInfo->binfo.pCtx, &w, &pInfo->timeWindowData, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pInfo->binfo.pCtx, &w, &pInfo->timeWindowData, startPos, 0, tsCols, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } // restore current time window - ret = setTimeWindowOutputBuf(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, + ret = setTimeWindowOutputBuf(pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { @@ -714,17 +718,17 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe #endif // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, + doWindowBorderInterpolation(pOperatorInfo, pBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, pInfo->order, false); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); STimeWindow nextWin = win; while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->order); if (startPos < 0) { break; } @@ -748,20 +752,20 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe ekey = ascScan? nextWin.ekey:nextWin.skey; forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, + doWindowBorderInterpolation(pOperatorInfo, pBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, pInfo->order, false); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } if (pInfo->timeWindowInterpo) { - int32_t rowIndex = ascScan ? (pSDataBlock->info.rows - 1) : 0; - saveDataBlockLastRow(pInfo->pRow, pSDataBlock->pDataBlock, rowIndex, pSDataBlock->info.numOfCols); + int32_t rowIndex = ascScan ? (pBlock->info.rows - 1) : 0; + saveDataBlockLastRow(pInfo->pRow, pBlock->pDataBlock, rowIndex, pBlock->info.numOfCols); } return pUpdated; @@ -815,9 +819,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { } closeAllResultRows(&pInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, - pInfo->binfo.rowCellInfoOffset); - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->order); OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; @@ -933,7 +934,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + if (pBInfo->pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; } @@ -941,8 +942,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - int32_t order = TSDB_ORDER_ASC; - STimeWindow win = pTaskInfo->window; + int32_t order = TSDB_ORDER_ASC; SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -955,18 +955,18 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { } setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, MAIN_SCAN, true); + blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId); + doStateWindowAggImpl(pOperator, pInfo, pBlock); } pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, - pBInfo->rowCellInfoOffset); initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + if (pBInfo->pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -994,7 +994,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + if (pBlock->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -1082,7 +1082,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + if (pInfo->binfo.pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; @@ -1413,7 +1413,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + if (pBInfo->pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; } @@ -1434,19 +1434,19 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, MAIN_SCAN, true); + blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId); + doSessionWindowAggImpl(pOperator, pInfo, pBlock); } // restore the value pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, - pBInfo->rowCellInfoOffset); initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + if (pBInfo->pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -1461,7 +1461,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) { STimeSliceOperatorInfo* pSliceInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { + if (pSliceInfo->binfo.pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pSliceInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -1493,7 +1493,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) { // initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); - if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { + if (pSliceInfo->binfo.pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pSliceInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -1695,7 +1695,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return NULL; } else if (pOperator->status == OP_RES_TO_RETURN) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + if (pInfo->binfo.pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 21fd54b620574272aad02315a3f0d74ae05405a8..00a9f3ae6c8ff087a9031c7c0d70e81bb3c88504 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -247,7 +247,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ return NULL; } - int32_t code = createDiskbasedBuf(&pHashObj->pBuf, pageSize, inMemPages * pageSize, 0, "/tmp"); + int32_t code = createDiskbasedBuf(&pHashObj->pBuf, pageSize, inMemPages * pageSize, 0, TD_TMP_DIR_PATH); if (code != 0) { terrno = code; return NULL; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index d585988e5e6c9802ee9864d89b956ecfbb2cdf33..c826cb68bfaa5053d6c6cf09aa409f4518df9dfc 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -155,7 +155,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { int32_t start = 0; if (pHandle->pBuf == NULL) { - int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "doAddToBuf", "/tmp"); + int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "doAddToBuf", TD_TMP_DIR_PATH); dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { return code; @@ -217,7 +217,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } else { // multi-pass internal merge sort is required if (pHandle->pBuf == NULL) { - code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "sortComparInit", "/tmp"); + code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "sortComparInit", TD_TMP_DIR_PATH); dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index c7ec0eece209a6d584ce846802abede47a9b8e81..880ac6c78eb8e04c08de7b612cdc58b50025e2c9 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -189,7 +189,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { pInfo->current += 1; - blockDataUpdateTsWindow(pBlock); + blockDataUpdateTsWindow(pBlock, 0); return pBlock; } diff --git a/source/libs/executor/test/index_executor_tests.cpp b/source/libs/executor/test/index_executor_tests.cpp index 2449bd1da1824ec94e4c071a0052950b29aeabe1..b0c2a983d1b5f60b50e4f5734a8c99fb3729d80e 100644 --- a/source/libs/executor/test/index_executor_tests.cpp +++ b/source/libs/executor/test/index_executor_tests.cpp @@ -57,7 +57,7 @@ void sifInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, "/tmp/sif"); + strcpy(tsLogDir, TD_TMP_DIR_PATH "sif"); taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 020df6cc3df28a424160c75e98e2900bdd38c0dc..ad92d095d5a292d366f127642e835b3dadda10dd 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -543,7 +543,7 @@ bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { } bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { - pEnv->calcMemSize = sizeof(double); + pEnv->calcMemSize = sizeof(SAvgRes); return true; } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index dd57024624f09d807996309c34e3990a39fdab46..90d0640f403ef3be0949f52d1313b715b225ced0 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -255,7 +255,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); - int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", "/tmp"); + int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", TD_TMP_DIR_PATH); if (ret != 0) { tMemBucketDestroy(pBucket); return NULL; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5f20d2e50a50fd0cef4e3b9cbaa21d22fb930464..d5dc967fed222fe95da9a853cff08de2ea8b440e 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -24,7 +24,6 @@ #include "builtinsimpl.h" #include "functionMgt.h" -//TODO: add unit test typedef struct SUdfdData { bool startCalled; bool needCleanUp; @@ -45,7 +44,15 @@ typedef struct SUdfdData { SUdfdData udfdGlobal = {0}; +int32_t udfStartUdfd(int32_t startDnodeId); +int32_t udfStopUdfd(); + static int32_t udfSpawnUdfd(SUdfdData *pData); +void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal); +static int32_t udfSpawnUdfd(SUdfdData* pData); +static void udfUdfdCloseWalkCb(uv_handle_t* handle, void* arg); +static void udfUdfdStopAsyncCb(uv_async_t *async); +static void udfWatchUdfd(void *args); void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { fnInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); @@ -413,6 +420,34 @@ enum { UDFC_STATE_STOPPING, // stopping after udfcClose }; +int32_t getUdfdPipeName(char* pipeName, int32_t size); +int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup); +void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request); +int32_t encodeUdfInterBuf(void **buf, const SUdfInterBuf* state); +void* decodeUdfInterBuf(const void* buf, SUdfInterBuf* state); +int32_t encodeUdfCallRequest(void **buf, const SUdfCallRequest *call); +void* decodeUdfCallRequest(const void* buf, SUdfCallRequest* call); +int32_t encodeUdfTeardownRequest(void **buf, const SUdfTeardownRequest *teardown); +void* decodeUdfTeardownRequest(const void* buf, SUdfTeardownRequest *teardown); +int32_t encodeUdfRequest(void** buf, const SUdfRequest* request); +void* decodeUdfRequest(const void* buf, SUdfRequest* request); +int32_t encodeUdfSetupResponse(void **buf, const SUdfSetupResponse *setupRsp); +void* decodeUdfSetupResponse(const void* buf, SUdfSetupResponse* setupRsp); +int32_t encodeUdfCallResponse(void **buf, const SUdfCallResponse *callRsp); +void* decodeUdfCallResponse(const void* buf, SUdfCallResponse* callRsp); +int32_t encodeUdfTeardownResponse(void** buf, const SUdfTeardownResponse* teardownRsp); +void* decodeUdfTeardownResponse(const void* buf, SUdfTeardownResponse* teardownResponse); +int32_t encodeUdfResponse(void** buf, const SUdfResponse* rsp); +void* decodeUdfResponse(const void* buf, SUdfResponse* rsp); +void freeUdfColumnData(SUdfColumnData *data, SUdfColumnMeta *meta); +void freeUdfColumn(SUdfColumn* col); +void freeUdfDataDataBlock(SUdfDataBlock *block); +void freeUdfInterBuf(SUdfInterBuf *buf); +int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock); +int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block); +int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output); +int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output); + int32_t getUdfdPipeName(char* pipeName, int32_t size) { char dnodeId[8] = {0}; size_t dnodeIdSize = sizeof(dnodeId); @@ -650,7 +685,7 @@ int32_t encodeUdfResponse(void** buf, const SUdfResponse* rsp) { len += encodeUdfTeardownResponse(buf, &rsp->teardownRsp); break; default: - //TODO: log error + fnError("encode udf response, invalid udf response type %d", rsp->type); break; } return len; @@ -676,7 +711,7 @@ void* decodeUdfResponse(const void* buf, SUdfResponse* rsp) { buf = decodeUdfTeardownResponse(buf, &rsp->teardownRsp); break; default: - //TODO: log error + fnError("decode udf response, invalid udf response type %d", rsp->type); break; } return (void*)buf; @@ -817,189 +852,499 @@ int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { return 0; } -void onUdfcPipeClose(uv_handle_t *handle) { - SClientUvConn *conn = handle->data; - if (!QUEUE_EMPTY(&conn->taskQueue)) { - QUEUE* h = QUEUE_HEAD(&conn->taskQueue); - SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); - task->errCode = 0; - QUEUE_REMOVE(&task->procTaskQueue); - uv_sem_post(&task->taskSem); - } - conn->session->udfUvPipe = NULL; - taosMemoryFree(conn->readBuf.buf); - taosMemoryFree(conn); - taosMemoryFree((uv_pipe_t *) handle); -} - -int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *uvTask) { - fnDebug("udfc get uv task result. task: %p, uvTask: %p", task, uvTask); - if (uvTask->type == UV_TASK_REQ_RSP) { - if (uvTask->rspBuf.base != NULL) { - SUdfResponse rsp = {0}; - void* buf = decodeUdfResponse(uvTask->rspBuf.base, &rsp); - assert(uvTask->rspBuf.len == POINTER_DISTANCE(buf, uvTask->rspBuf.base)); - task->errCode = rsp.code; +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +//memory layout |---SUdfAggRes----|-----final result-----|---inter result----| +typedef struct SUdfAggRes { + int8_t finalResNum; + int8_t interResNum; + char* finalResBuf; + char* interResBuf; +} SUdfAggRes; +void onUdfcPipeClose(uv_handle_t *handle); +int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *uvTask); +void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf); +bool isUdfcUvMsgComplete(SClientConnBuf *connBuf); +void udfcUvHandleRsp(SClientUvConn *conn); +void udfcUvHandleError(SClientUvConn *conn); +void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); +void onUdfcPipetWrite(uv_write_t *write, int status); +void onUdfcPipeConnect(uv_connect_t *connect, int status); +int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask); +int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask); +int32_t udfcStartUvTask(SClientUvTaskNode *uvTask); +void udfcAsyncTaskCb(uv_async_t *async); +void cleanUpUvTasks(SUdfcProxy *udfc); +void udfStopAsyncCb(uv_async_t *async); +void constructUdfService(void *argsThread); +int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType); +int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle); +int compareUdfcFuncSub(const void* elem1, const void* elem2); +int32_t doTeardownUdf(UdfcFuncHandle handle); - switch (task->type) { - case UDF_TASK_SETUP: { - //TODO: copy or not - task->_setup.rsp = rsp.setupRsp; - break; - } - case UDF_TASK_CALL: { - task->_call.rsp = rsp.callRsp; - //TODO: copy or not - break; - } - case UDF_TASK_TEARDOWN: { - task->_teardown.rsp = rsp.teardownRsp; - //TODO: copy or not? - break; - } - default: { - break; - } - } +int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, + SSDataBlock* output, SUdfInterBuf *newState); +int32_t doCallUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf); +int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); +int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); +int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); +int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam* output); +int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output); + +int32_t udfcOpen(); +int32_t udfcClose(); + +int32_t acquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle); +void releaseUdfFuncHandle(char* udfName); +int32_t cleanUpUdfs(); + +bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); +int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); +int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); - // TODO: the call buffer is setup and freed by udf invocation - taosMemoryFree(uvTask->rspBuf.base); - } else { - task->errCode = uvTask->errCode; - } - } else if (uvTask->type == UV_TASK_CONNECT) { - task->errCode = uvTask->errCode; - } else if (uvTask->type == UV_TASK_DISCONNECT) { - task->errCode = uvTask->errCode; - } - return 0; +int compareUdfcFuncSub(const void* elem1, const void* elem2) { + SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1; + SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2; + return strcmp(stub1->udfName, stub2->udfName); } -void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { - SClientUvConn *conn = handle->data; - SClientConnBuf *connBuf = &conn->readBuf; - - int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); - if (connBuf->cap == 0) { - connBuf->buf = taosMemoryMalloc(msgHeadSize); - if (connBuf->buf) { - connBuf->len = 0; - connBuf->cap = msgHeadSize; - connBuf->total = -1; - - buf->base = connBuf->buf; - buf->len = connBuf->cap; +int32_t acquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) { + int32_t code = 0; + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); + SUdfcFuncStub key = {0}; + strcpy(key.udfName, udfName); + int32_t stubIndex = taosArraySearchIdx(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); + if (stubIndex != -1) { + SUdfcFuncStub *foundStub = taosArrayGet(gUdfdProxy.udfStubs, stubIndex); + UdfcFuncHandle handle = foundStub->handle; + if (handle != NULL && ((SUdfcUvSession*)handle)->udfUvPipe != NULL) { + *pHandle = foundStub->handle; + ++foundStub->refCount; + foundStub->lastRefTime = taosGetTimestampUs(); + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); + return 0; } else { - fnError("udfc allocate buffer failure. size: %d", msgHeadSize); - buf->base = NULL; - buf->len = 0; + fnInfo("invalid handle for %s, refCount: %d, last ref time: %"PRId64". remove it from cache", + udfName, foundStub->refCount, foundStub->lastRefTime); + taosArrayRemove(gUdfdProxy.udfStubs, stubIndex); } + } + *pHandle = NULL; + code = doSetupUdf(udfName, pHandle); + if (code == TSDB_CODE_SUCCESS) { + SUdfcFuncStub stub = {0}; + strcpy(stub.udfName, udfName); + stub.handle = *pHandle; + ++stub.refCount; + stub.lastRefTime = taosGetTimestampUs(); + taosArrayPush(gUdfdProxy.udfStubs, &stub); + taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub); } else { - connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap; - void *resultBuf = taosMemoryRealloc(connBuf->buf, connBuf->cap); - if (resultBuf) { - connBuf->buf = resultBuf; - buf->base = connBuf->buf + connBuf->len; - buf->len = connBuf->cap - connBuf->len; - } else { - fnError("udfc re-allocate buffer failure. size: %d", connBuf->cap); - buf->base = NULL; - buf->len = 0; - } + *pHandle = NULL; } - fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); - + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); + return code; } -bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { - if (connBuf->total == -1 && connBuf->len >= sizeof(int32_t)) { - connBuf->total = *(int32_t *) (connBuf->buf); +void releaseUdfFuncHandle(char* udfName) { + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); + SUdfcFuncStub key = {0}; + strcpy(key.udfName, udfName); + SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); + if (!foundStub) { + return; } - if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { - fnTrace("udfc complete message is received, now handle it"); - return true; + if (foundStub->refCount > 0) { + --foundStub->refCount; } - return false; + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); } -void udfcUvHandleRsp(SClientUvConn *conn) { - SClientConnBuf *connBuf = &conn->readBuf; - int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum - - if (QUEUE_EMPTY(&conn->taskQueue)) { - fnError("udfc no task waiting for response on connection"); - return; - } - bool found = false; - SClientUvTaskNode *taskFound = NULL; - QUEUE* h = QUEUE_NEXT(&conn->taskQueue); - SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); - - while (h != &conn->taskQueue) { - if (task->seqNum == seqNum) { - if (found == false) { - found = true; - taskFound = task; +int32_t cleanUpUdfs() { + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); + int32_t i = 0; + SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); + while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) { + SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i); + if (stub->refCount == 0) { + fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); + doTeardownUdf(stub->handle); + } else { + fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p", + stub->udfName, stub->refCount, stub->lastRefTime, stub->handle); + UdfcFuncHandle handle = stub->handle; + if (handle != NULL && ((SUdfcUvSession*)handle)->udfUvPipe != NULL) { + taosArrayPush(udfStubs, stub); } else { - fnError("udfc more than one task waiting for the same response"); - continue; + fnInfo("udf invalid handle for %s, refCount: %d, last ref time: %"PRId64". remove it from cache", + stub->udfName, stub->refCount, stub->lastRefTime); } } - h = QUEUE_NEXT(h); - task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); + ++i; } + taosArrayDestroy(gUdfdProxy.udfStubs); + gUdfdProxy.udfStubs = udfStubs; + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); + return 0; +} - if (taskFound) { - taskFound->rspBuf = uv_buf_init(connBuf->buf, connBuf->len); - QUEUE_REMOVE(&taskFound->connTaskQueue); - QUEUE_REMOVE(&taskFound->procTaskQueue); - uv_sem_post(&taskFound->taskSem); +int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) { + UdfcFuncHandle handle = NULL; + int32_t code = acquireUdfFuncHandle(udfName, &handle); + if (code != 0) { + return code; + } + SUdfcUvSession *session = handle; + code = doCallUdfScalarFunc(handle, input, numOfCols, output); + if (output->columnData == NULL) { + fnError("udfc scalar function calculate error. no column data"); + code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; } else { - fnError("no task is waiting for the response."); + if (session->outputType != output->columnData->info.type || session->outputLen != output->columnData->info.bytes) { + fnError("udfc scalar function calculate error. type mismatch. session type: %d(%d), output type: %d(%d)", session->outputType, + session->outputLen, output->columnData->info.type, output->columnData->info.bytes); + code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } } - connBuf->buf = NULL; - connBuf->total = -1; - connBuf->len = 0; - connBuf->cap = 0; + releaseUdfFuncHandle(udfName); + return code; } -void udfcUvHandleError(SClientUvConn *conn) { - while (!QUEUE_EMPTY(&conn->taskQueue)) { - QUEUE* h = QUEUE_HEAD(&conn->taskQueue); - SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); - task->errCode = TSDB_CODE_UDF_PIPE_READ_ERR; - QUEUE_REMOVE(&task->connTaskQueue); - QUEUE_REMOVE(&task->procTaskQueue); - uv_sem_post(&task->taskSem); +bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + if (fmIsScalarFunc(pFunc->funcId)) { + return false; } - - uv_close((uv_handle_t *) conn->pipe, onUdfcPipeClose); + pEnv->calcMemSize = sizeof(SUdfAggRes) + pFunc->node.resType.bytes + pFunc->udfBufSize; + return true; } -void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread); - if (nread == 0) return; - - SClientUvConn *conn = client->data; - SClientConnBuf *connBuf = &conn->readBuf; - if (nread > 0) { - connBuf->len += nread; - if (isUdfcUvMsgComplete(connBuf)) { - udfcUvHandleRsp(conn); - } - - } - if (nread < 0) { - fnError("udfc client pipe %p read error: %zd, %s.", client, nread, uv_strerror(nread)); - if (nread == UV_EOF) { - fnError("\tudfc client pipe %p closed", client); - } - udfcUvHandleError(conn); +bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo) { + if (functionSetup(pCtx, pResultCellInfo) != true) { + return false; } + UdfcFuncHandle handle; + int32_t udfCode = 0; + if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode); + return false; + } + SUdfcUvSession *session = (SUdfcUvSession *)handle; + SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo); + int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; + memset(udfRes, 0, envSize); + + udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); + udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + + SUdfInterBuf buf = {0}; + if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { + fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); + releaseUdfFuncHandle(pCtx->udfName); + return false; + } + udfRes->interResNum = buf.numOfResult; + if (buf.bufLen <= session->bufSize) { + memcpy(udfRes->interResBuf, buf.buf, buf.bufLen); + } else { + fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); + releaseUdfFuncHandle(pCtx->udfName); + return false; + } + releaseUdfFuncHandle(pCtx->udfName); + freeUdfInterBuf(&buf); + return true; +} + +int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { + int32_t udfCode = 0; + UdfcFuncHandle handle = 0; + if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); + return udfCode; + } + + SUdfcUvSession *session = handle; + SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); + udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + + SInputColumnInfoData* pInput = &pCtx->input; + int32_t numOfCols = pInput->numOfInputCols; + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + + SSDataBlock tempBlock = {0}; + tempBlock.info.numOfCols = numOfCols; + tempBlock.info.rows = pInput->totalRows; + tempBlock.info.uid = pInput->uid; + bool hasVarCol = false; + tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData *col = pInput->pData[i]; + if (IS_VAR_DATA_TYPE(col->info.type)) { + hasVarCol = true; + } + taosArrayPush(tempBlock.pDataBlock, col); + } + tempBlock.info.hasVarCol = hasVarCol; + + SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows); + + SUdfInterBuf state = {.buf = udfRes->interResBuf, + .bufLen = session->bufSize, + .numOfResult = udfRes->interResNum}; + SUdfInterBuf newState = {0}; + + udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); + if (udfCode != 0) { + fnError("udfAggProcess error. code: %d", udfCode); + newState.numOfResult = 0; + } else { + udfRes->interResNum = newState.numOfResult; + if (newState.bufLen <= session->bufSize) { + memcpy(udfRes->interResBuf, newState.buf, newState.bufLen); + } else { + fnError("udfc inter buf size %d is greater than function bufSize %d", newState.bufLen, session->bufSize); + udfCode = TSDB_CODE_UDF_INVALID_BUFSIZE; + } + } + if (newState.numOfResult == 1 || state.numOfResult == 1) { + GET_RES_INFO(pCtx)->numOfRes = 1; + } + + blockDataDestroy(inputBlock); + taosArrayDestroy(tempBlock.pDataBlock); + + releaseUdfFuncHandle(pCtx->udfName); + freeUdfInterBuf(&newState); + return udfCode; +} + +int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { + int32_t udfCode = 0; + UdfcFuncHandle handle = 0; + if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); + return udfCode; + } + + SUdfcUvSession *session = handle; + SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); + udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + + + SUdfInterBuf resultBuf = {0}; + SUdfInterBuf state = {.buf = udfRes->interResBuf, + .bufLen = session->bufSize, + .numOfResult = udfRes->interResNum}; + int32_t udfCallCode= 0; + udfCallCode= doCallUdfAggFinalize(session, &state, &resultBuf); + if (udfCallCode != 0) { + fnError("udfAggFinalize error. doCallUdfAggFinalize step. udf code:%d", udfCallCode); + GET_RES_INFO(pCtx)->numOfRes = 0; + } else { + if (resultBuf.bufLen <= session->outputLen) { + memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen); + udfRes->finalResNum = resultBuf.numOfResult; + GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum; + } else { + fnError("udfc inter buf size %d is greater than function output size %d", resultBuf.bufLen, session->outputLen); + GET_RES_INFO(pCtx)->numOfRes = 0; + udfCallCode = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } + } + + freeUdfInterBuf(&resultBuf); + + int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); + releaseUdfFuncHandle(pCtx->udfName); + return udfCallCode == 0 ? numOfResults : udfCallCode; +} + +void onUdfcPipeClose(uv_handle_t *handle) { + SClientUvConn *conn = handle->data; + if (!QUEUE_EMPTY(&conn->taskQueue)) { + QUEUE* h = QUEUE_HEAD(&conn->taskQueue); + SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); + task->errCode = 0; + QUEUE_REMOVE(&task->procTaskQueue); + uv_sem_post(&task->taskSem); + } + conn->session->udfUvPipe = NULL; + taosMemoryFree(conn->readBuf.buf); + taosMemoryFree(conn); + taosMemoryFree((uv_pipe_t *) handle); +} + +int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *uvTask) { + fnDebug("udfc get uv task result. task: %p, uvTask: %p", task, uvTask); + if (uvTask->type == UV_TASK_REQ_RSP) { + if (uvTask->rspBuf.base != NULL) { + SUdfResponse rsp = {0}; + void* buf = decodeUdfResponse(uvTask->rspBuf.base, &rsp); + assert(uvTask->rspBuf.len == POINTER_DISTANCE(buf, uvTask->rspBuf.base)); + task->errCode = rsp.code; + + switch (task->type) { + case UDF_TASK_SETUP: { + task->_setup.rsp = rsp.setupRsp; + break; + } + case UDF_TASK_CALL: { + task->_call.rsp = rsp.callRsp; + break; + } + case UDF_TASK_TEARDOWN: { + task->_teardown.rsp = rsp.teardownRsp; + break; + } + default: { + break; + } + } + + // TODO: the call buffer is setup and freed by udf invocation + taosMemoryFree(uvTask->rspBuf.base); + } else { + task->errCode = uvTask->errCode; + } + } else if (uvTask->type == UV_TASK_CONNECT) { + task->errCode = uvTask->errCode; + } else if (uvTask->type == UV_TASK_DISCONNECT) { + task->errCode = uvTask->errCode; + } + return 0; +} + +void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { + SClientUvConn *conn = handle->data; + SClientConnBuf *connBuf = &conn->readBuf; + + int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); + if (connBuf->cap == 0) { + connBuf->buf = taosMemoryMalloc(msgHeadSize); + if (connBuf->buf) { + connBuf->len = 0; + connBuf->cap = msgHeadSize; + connBuf->total = -1; + + buf->base = connBuf->buf; + buf->len = connBuf->cap; + } else { + fnError("udfc allocate buffer failure. size: %d", msgHeadSize); + buf->base = NULL; + buf->len = 0; + } + } else { + connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap; + void *resultBuf = taosMemoryRealloc(connBuf->buf, connBuf->cap); + if (resultBuf) { + connBuf->buf = resultBuf; + buf->base = connBuf->buf + connBuf->len; + buf->len = connBuf->cap - connBuf->len; + } else { + fnError("udfc re-allocate buffer failure. size: %d", connBuf->cap); + buf->base = NULL; + buf->len = 0; + } + } + + fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); + +} + +bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { + if (connBuf->total == -1 && connBuf->len >= sizeof(int32_t)) { + connBuf->total = *(int32_t *) (connBuf->buf); + } + if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + fnTrace("udfc complete message is received, now handle it"); + return true; + } + return false; +} + +void udfcUvHandleRsp(SClientUvConn *conn) { + SClientConnBuf *connBuf = &conn->readBuf; + int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum + + if (QUEUE_EMPTY(&conn->taskQueue)) { + fnError("udfc no task waiting for response on connection"); + return; + } + bool found = false; + SClientUvTaskNode *taskFound = NULL; + QUEUE* h = QUEUE_NEXT(&conn->taskQueue); + SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); + + while (h != &conn->taskQueue) { + if (task->seqNum == seqNum) { + if (found == false) { + found = true; + taskFound = task; + } else { + fnError("udfc more than one task waiting for the same response"); + continue; + } + } + h = QUEUE_NEXT(h); + task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); + } + + if (taskFound) { + taskFound->rspBuf = uv_buf_init(connBuf->buf, connBuf->len); + QUEUE_REMOVE(&taskFound->connTaskQueue); + QUEUE_REMOVE(&taskFound->procTaskQueue); + uv_sem_post(&taskFound->taskSem); + } else { + fnError("no task is waiting for the response."); + } + connBuf->buf = NULL; + connBuf->total = -1; + connBuf->len = 0; + connBuf->cap = 0; +} + +void udfcUvHandleError(SClientUvConn *conn) { + while (!QUEUE_EMPTY(&conn->taskQueue)) { + QUEUE* h = QUEUE_HEAD(&conn->taskQueue); + SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue); + task->errCode = TSDB_CODE_UDF_PIPE_READ_ERR; + QUEUE_REMOVE(&task->connTaskQueue); + QUEUE_REMOVE(&task->procTaskQueue); + uv_sem_post(&task->taskSem); + } + + uv_close((uv_handle_t *) conn->pipe, onUdfcPipeClose); +} + +void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { + fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread); + if (nread == 0) return; + + SClientUvConn *conn = client->data; + SClientConnBuf *connBuf = &conn->readBuf; + if (nread > 0) { + connBuf->len += nread; + if (isUdfcUvMsgComplete(connBuf)) { + udfcUvHandleRsp(conn); + } + + } + if (nread < 0) { + fnError("udfc client pipe %p read error: %zd, %s.", client, nread, uv_strerror(nread)); + if (nread == UV_EOF) { + fnError("\tudfc client pipe %p closed", client); + } + udfcUvHandleError(conn); + } + +} -} - void onUdfcPipetWrite(uv_write_t *write, int status) { SClientUvTaskNode *uvTask = write->data; uv_pipe_t *pipe = uvTask->pipe; @@ -1050,7 +1395,7 @@ int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN request.teardown = task->_teardown.req; request.type = UDF_TASK_TEARDOWN; } else { - //TODO log and return error + fnError("udfc create uv task, invalid task type : %d", task->type); } int32_t bufLen = encodeUdfRequest(NULL, &request); request.msgLen = bufLen; @@ -1271,134 +1616,47 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { SClientUvConn *conn = uvTask->pipe->data; conn->session = task->session; } - taosMemoryFree(uvTask); - uvTask = NULL; - return task->errCode; -} - -int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { - if (gUdfdProxy.udfcState != UDFC_STATE_READY) { - return TSDB_CODE_UDF_INVALID_STATE; - } - SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask)); - task->errCode = 0; - task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession)); - task->session->udfc = &gUdfdProxy; - task->type = UDF_TASK_SETUP; - - SUdfSetupRequest *req = &task->_setup.req; - strncpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN); - - int32_t errCode = udfcRunUdfUvTask(task, UV_TASK_CONNECT); - if (errCode != 0) { - fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfdProxy)->udfdPipeName); - return TSDB_CODE_UDF_PIPE_CONNECT_ERR; - } - - udfcRunUdfUvTask(task, UV_TASK_REQ_RSP); - - SUdfSetupResponse *rsp = &task->_setup.rsp; - task->session->severHandle = rsp->udfHandle; - task->session->outputType = rsp->outputType; - task->session->outputLen = rsp->outputLen; - task->session->bufSize = rsp->bufSize; - strcpy(task->session->udfName, udfName); - if (task->errCode != 0) { - fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode) - } else { - fnInfo("sucessfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session); - *funcHandle = task->session; - } - int32_t err = task->errCode; - taosMemoryFree(task); - return err; -} - -int compareUdfcFuncSub(const void* elem1, const void* elem2) { - SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1; - SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2; - return strcmp(stub1->udfName, stub2->udfName); -} - -int32_t acquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) { - int32_t code = 0; - uv_mutex_lock(&gUdfdProxy.udfStubsMutex); - SUdfcFuncStub key = {0}; - strcpy(key.udfName, udfName); - int32_t stubIndex = taosArraySearchIdx(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); - if (stubIndex != -1) { - SUdfcFuncStub *foundStub = taosArrayGet(gUdfdProxy.udfStubs, stubIndex); - UdfcFuncHandle handle = foundStub->handle; - if (handle != NULL && ((SUdfcUvSession*)handle)->udfUvPipe != NULL) { - *pHandle = foundStub->handle; - ++foundStub->refCount; - foundStub->lastRefTime = taosGetTimestampUs(); - uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); - return 0; - } else { - fnInfo("invalid handle for %s, refCount: %d, last ref time: %"PRId64". remove it from cache", - udfName, foundStub->refCount, foundStub->lastRefTime); - taosArrayRemove(gUdfdProxy.udfStubs, stubIndex); - } - } - *pHandle = NULL; - code = doSetupUdf(udfName, pHandle); - if (code == TSDB_CODE_SUCCESS) { - SUdfcFuncStub stub = {0}; - strcpy(stub.udfName, udfName); - stub.handle = *pHandle; - ++stub.refCount; - stub.lastRefTime = taosGetTimestampUs(); - taosArrayPush(gUdfdProxy.udfStubs, &stub); - taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub); - } else { - *pHandle = NULL; - } - - uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); - return code; + taosMemoryFree(uvTask); + uvTask = NULL; + return task->errCode; } -void releaseUdfFuncHandle(char* udfName) { - uv_mutex_lock(&gUdfdProxy.udfStubsMutex); - SUdfcFuncStub key = {0}; - strcpy(key.udfName, udfName); - SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); - if (!foundStub) { - return; +int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { + if (gUdfdProxy.udfcState != UDFC_STATE_READY) { + return TSDB_CODE_UDF_INVALID_STATE; } - if (foundStub->refCount > 0) { - --foundStub->refCount; + SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession)); + task->session->udfc = &gUdfdProxy; + task->type = UDF_TASK_SETUP; + + SUdfSetupRequest *req = &task->_setup.req; + strncpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN); + + int32_t errCode = udfcRunUdfUvTask(task, UV_TASK_CONNECT); + if (errCode != 0) { + fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfdProxy)->udfdPipeName); + return TSDB_CODE_UDF_PIPE_CONNECT_ERR; } - uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); -} -int32_t cleanUpUdfs() { - uv_mutex_lock(&gUdfdProxy.udfStubsMutex); - int32_t i = 0; - SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); - while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) { - SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i); - if (stub->refCount == 0) { - fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); - doTeardownUdf(stub->handle); - } else { - fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p", - stub->udfName, stub->refCount, stub->lastRefTime, stub->handle); - UdfcFuncHandle handle = stub->handle; - if (handle != NULL && ((SUdfcUvSession*)handle)->udfUvPipe != NULL) { - taosArrayPush(udfStubs, stub); - } else { - fnInfo("udf invalid handle for %s, refCount: %d, last ref time: %"PRId64". remove it from cache", - stub->udfName, stub->refCount, stub->lastRefTime); - } - } - ++i; + udfcRunUdfUvTask(task, UV_TASK_REQ_RSP); + + SUdfSetupResponse *rsp = &task->_setup.rsp; + task->session->severHandle = rsp->udfHandle; + task->session->outputType = rsp->outputType; + task->session->outputLen = rsp->outputLen; + task->session->bufSize = rsp->bufSize; + strcpy(task->session->udfName, udfName); + if (task->errCode != 0) { + fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode) + } else { + fnInfo("sucessfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session); + *funcHandle = task->session; } - taosArrayDestroy(gUdfdProxy.udfStubs); - gUdfdProxy.udfStubs = udfStubs; - uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); - return 0; + int32_t err = task->errCode; + taosMemoryFree(task); + return err; } int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, @@ -1524,29 +1782,6 @@ int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t return err; } - -int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) { - UdfcFuncHandle handle = NULL; - int32_t code = acquireUdfFuncHandle(udfName, &handle); - if (code != 0) { - return code; - } - SUdfcUvSession *session = handle; - code = doCallUdfScalarFunc(handle, input, numOfCols, output); - if (output->columnData == NULL) { - fnError("udfc scalar function calculate error. no column data"); - code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; - } else { - if (session->outputType != output->columnData->info.type || session->outputLen != output->columnData->info.bytes) { - fnError("udfc scalar function calculate error. type mismatch. session type: %d(%d), output type: %d(%d)", session->outputType, - session->outputLen, output->columnData->info.type, output->columnData->info.bytes); - code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; - } - } - releaseUdfFuncHandle(udfName); - return code; -} - int32_t doTeardownUdf(UdfcFuncHandle handle) { SUdfcUvSession *session = (SUdfcUvSession *) handle; @@ -1576,165 +1811,3 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { return err; } - -//memory layout |---SUdfAggRes----|-----final result-----|---inter result----| -typedef struct SUdfAggRes { - int8_t finalResNum; - int8_t interResNum; - char* finalResBuf; - char* interResBuf; -} SUdfAggRes; - -bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - if (fmIsScalarFunc(pFunc->funcId)) { - return false; - } - pEnv->calcMemSize = sizeof(SUdfAggRes) + pFunc->node.resType.bytes + pFunc->udfBufSize; - return true; -} - -bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo) { - if (functionSetup(pCtx, pResultCellInfo) != true) { - return false; - } - UdfcFuncHandle handle; - int32_t udfCode = 0; - if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { - fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode); - return false; - } - SUdfcUvSession *session = (SUdfcUvSession *)handle; - SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo); - int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; - memset(udfRes, 0, envSize); - - udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - - SUdfInterBuf buf = {0}; - if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { - fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); - releaseUdfFuncHandle(pCtx->udfName); - return false; - } - udfRes->interResNum = buf.numOfResult; - if (buf.bufLen <= session->bufSize) { - memcpy(udfRes->interResBuf, buf.buf, buf.bufLen); - } else { - fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); - releaseUdfFuncHandle(pCtx->udfName); - return false; - } - releaseUdfFuncHandle(pCtx->udfName); - freeUdfInterBuf(&buf); - return true; -} - -int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { - int32_t udfCode = 0; - UdfcFuncHandle handle = 0; - if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { - fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); - return udfCode; - } - - SUdfcUvSession *session = handle; - SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - - SInputColumnInfoData* pInput = &pCtx->input; - int32_t numOfCols = pInput->numOfInputCols; - int32_t start = pInput->startRowIndex; - int32_t numOfRows = pInput->numOfRows; - - - SSDataBlock tempBlock = {0}; - tempBlock.info.numOfCols = numOfCols; - tempBlock.info.rows = pInput->totalRows; - tempBlock.info.uid = pInput->uid; - bool hasVarCol = false; - tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *col = pInput->pData[i]; - if (IS_VAR_DATA_TYPE(col->info.type)) { - hasVarCol = true; - } - taosArrayPush(tempBlock.pDataBlock, col); - } - tempBlock.info.hasVarCol = hasVarCol; - - SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows); - - SUdfInterBuf state = {.buf = udfRes->interResBuf, - .bufLen = session->bufSize, - .numOfResult = udfRes->interResNum}; - SUdfInterBuf newState = {0}; - - udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); - if (udfCode != 0) { - fnError("udfAggProcess error. code: %d", udfCode); - newState.numOfResult = 0; - } else { - udfRes->interResNum = newState.numOfResult; - if (newState.bufLen <= session->bufSize) { - memcpy(udfRes->interResBuf, newState.buf, newState.bufLen); - } else { - fnError("udfc inter buf size %d is greater than function bufSize %d", newState.bufLen, session->bufSize); - udfCode = TSDB_CODE_UDF_INVALID_BUFSIZE; - } - } - if (newState.numOfResult == 1 || state.numOfResult == 1) { - GET_RES_INFO(pCtx)->numOfRes = 1; - } - - blockDataDestroy(inputBlock); - taosArrayDestroy(tempBlock.pDataBlock); - - releaseUdfFuncHandle(pCtx->udfName); - freeUdfInterBuf(&newState); - return udfCode; -} - -int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { - int32_t udfCode = 0; - UdfcFuncHandle handle = 0; - if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { - fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); - return udfCode; - } - - SUdfcUvSession *session = handle; - SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - - - SUdfInterBuf resultBuf = {0}; - SUdfInterBuf state = {.buf = udfRes->interResBuf, - .bufLen = session->bufSize, - .numOfResult = udfRes->interResNum}; - int32_t udfCallCode= 0; - udfCallCode= doCallUdfAggFinalize(session, &state, &resultBuf); - if (udfCallCode != 0) { - fnError("udfAggFinalize error. doCallUdfAggFinalize step. udf code:%d", udfCallCode); - GET_RES_INFO(pCtx)->numOfRes = 0; - } else { - if (resultBuf.bufLen <= session->outputLen) { - memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen); - udfRes->finalResNum = resultBuf.numOfResult; - GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum; - } else { - fnError("udfc inter buf size %d is greater than function output size %d", resultBuf.bufLen, session->outputLen); - GET_RES_INFO(pCtx)->numOfRes = 0; - udfCallCode = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; - } - } - - freeUdfInterBuf(&resultBuf); - - int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); - releaseUdfFuncHandle(pCtx->udfName); - return udfCallCode == 0 ? numOfResults : udfCallCode; -} diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 9185f707116b54c8493843a3a71bd343ae4ac4d0..fa808582533f934216a146bcf13fcd73caa29831 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -103,6 +103,262 @@ typedef struct SUdfdRpcSendRecvInfo { uv_sem_t resultSem; } SUdfdRpcSendRecvInfo; +static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); +static int32_t udfdConnectToMnode(); +static int32_t udfdLoadUdf(char *udfName, SUdf *udf); +static bool udfdRpcRfp(int32_t code); +static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); +static int32_t udfdOpenClientRpc(); +static int32_t udfdCloseClientRpc(); + +static void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request); +static void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request); +static void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request); +static void udfdProcessRequest(uv_work_t *req); +static void udfdOnWrite(uv_write_t *req, int status); +static void udfdSendResponse(uv_work_t *work, int status); +static void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf); +static bool isUdfdUvMsgComplete(SUdfdUvConn *pipe); +static void udfdHandleRequest(SUdfdUvConn *conn); +static void udfdPipeCloseCb(uv_handle_t *pipe); +static void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn->client, udfdPipeCloseCb); } +static void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); +static void udfdOnNewConnection(uv_stream_t *server, int status); + +static void udfdIntrSignalHandler(uv_signal_t *handle, int signum); +static int32_t removeListeningPipe(); + +static void udfdPrintVersion(); +static int32_t udfdParseArgs(int32_t argc, char *argv[]); +static int32_t udfdInitLog(); + +static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf); +static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf); +static int32_t udfdUvInit(); +static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); +static int32_t udfdRun(); + +void udfdProcessRequest(uv_work_t *req) { + SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); + SUdfRequest request = {0}; + decodeUdfRequest(uvUdf->input.base, &request); + + switch (request.type) { + case UDF_TASK_SETUP: { + udfdProcessSetupRequest(uvUdf, &request); + break; + } + + case UDF_TASK_CALL: { + udfdProcessCallRequest(uvUdf, &request); + break; + } + case UDF_TASK_TEARDOWN: { + udfdProcessTeardownRequest(uvUdf, &request); + break; + } + default: { + break; + } + } +} + +void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { + // TODO: tracable id from client. connect, setup, call, teardown + fnInfo("setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName); + SUdfSetupRequest *setup = &request->setup; + int32_t code = TSDB_CODE_SUCCESS; + SUdf * udf = NULL; + uv_mutex_lock(&global.udfsMutex); + SUdf **udfInHash = taosHashGet(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName)); + if (udfInHash) { + ++(*udfInHash)->refCount; + udf = *udfInHash; + uv_mutex_unlock(&global.udfsMutex); + } else { + SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); + udfNew->refCount = 1; + udfNew->state = UDF_STATE_INIT; + + uv_mutex_init(&udfNew->lock); + uv_cond_init(&udfNew->condReady); + udf = udfNew; + taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), &udfNew, sizeof(&udfNew)); + uv_mutex_unlock(&global.udfsMutex); + } + + uv_mutex_lock(&udf->lock); + if (udf->state == UDF_STATE_INIT) { + udf->state = UDF_STATE_LOADING; + code = udfdLoadUdf(setup->udfName, udf); + if (udf->initFunc) { + udf->initFunc(); + } + udf->state = UDF_STATE_READY; + uv_cond_broadcast(&udf->condReady); + uv_mutex_unlock(&udf->lock); + } else { + while (udf->state != UDF_STATE_READY) { + uv_cond_wait(&udf->condReady, &udf->lock); + } + uv_mutex_unlock(&udf->lock); + } + SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); + handle->udf = udf; + + SUdfResponse rsp; + rsp.seqNum = request->seqNum; + rsp.type = request->type; + rsp.code = code; + rsp.setupRsp.udfHandle = (int64_t)(handle); + rsp.setupRsp.outputType = udf->outputType; + rsp.setupRsp.outputLen = udf->outputLen; + rsp.setupRsp.bufSize = udf->bufSize; + + int32_t len = encodeUdfResponse(NULL, &rsp); + rsp.msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, &rsp); + + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + return; +} + +void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { + SUdfCallRequest *call = &request->call; + fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request->seqNum, call->callType, call->udfHandle); + SUdfcFuncHandle * handle = (SUdfcFuncHandle *)(call->udfHandle); + SUdf * udf = handle->udf; + SUdfResponse response = {0}; + SUdfResponse * rsp = &response; + SUdfCallResponse *subRsp = &rsp->callRsp; + + int32_t code = TSDB_CODE_SUCCESS; + switch (call->callType) { + case TSDB_UDF_CALL_SCALA_PROC: { + SUdfColumn output = {0}; + + SUdfDataBlock input = {0}; + convertDataBlockToUdfDataBlock(&call->block, &input); + code = udf->scalarProcFunc(&input, &output); + freeUdfDataDataBlock(&input); + convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); + freeUdfColumn(&output); + break; + } + case TSDB_UDF_CALL_AGG_INIT: { + SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; + udf->aggStartFunc(&outBuf); + subRsp->resultBuf = outBuf; + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + SUdfDataBlock input = {0}; + convertDataBlockToUdfDataBlock(&call->block, &input); + SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; + code = udf->aggProcFunc(&input, &call->interBuf, &outBuf); + freeUdfInterBuf(&call->interBuf); + freeUdfDataDataBlock(&input); + subRsp->resultBuf = outBuf; + + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; + code = udf->aggFinishFunc(&call->interBuf, &outBuf); + freeUdfInterBuf(&call->interBuf); + subRsp->resultBuf = outBuf; + break; + } + default: + break; + } + + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = code; + subRsp->callType = call->callType; + + int32_t len = encodeUdfResponse(NULL, rsp); + rsp->msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, rsp); + uvUdf->output = uv_buf_init(bufBegin, len); + + switch (call->callType) { + case TSDB_UDF_CALL_SCALA_PROC: { + tDeleteSSDataBlock(&call->block); + tDeleteSSDataBlock(&subRsp->resultData); + break; + } + case TSDB_UDF_CALL_AGG_INIT: { + freeUdfInterBuf(&subRsp->resultBuf); + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + tDeleteSSDataBlock(&call->block); + freeUdfInterBuf(&subRsp->resultBuf); + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + freeUdfInterBuf(&subRsp->resultBuf); + break; + } + default: + break; + } + + taosMemoryFree(uvUdf->input.base); + return; +} + +void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { + SUdfTeardownRequest *teardown = &request->teardown; + fnInfo("teardown. seq number: %" PRId64 ", handle:%" PRIx64, request->seqNum, teardown->udfHandle); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); + SUdf * udf = handle->udf; + bool unloadUdf = false; + int32_t code = TSDB_CODE_SUCCESS; + + uv_mutex_lock(&global.udfsMutex); + udf->refCount--; + if (udf->refCount == 0) { + unloadUdf = true; + taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); + } + uv_mutex_unlock(&global.udfsMutex); + if (unloadUdf) { + uv_cond_destroy(&udf->condReady); + uv_mutex_destroy(&udf->lock); + if (udf->destroyFunc) { + (udf->destroyFunc)(); + } + uv_dlclose(&udf->lib); + taosMemoryFree(udf); + } + taosMemoryFree(handle); + + SUdfResponse response; + SUdfResponse *rsp = &response; + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = code; + int32_t len = encodeUdfResponse(NULL, rsp); + rsp->msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, rsp); + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + return; +} + void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->info.ahandle; ASSERT(pMsg->info.ahandle != NULL); @@ -144,11 +400,14 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { udf->bufSize = pFuncInfo->bufSize; char path[PATH_MAX] = {0}; - snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", pFuncInfo->name); + snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name); TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); - // TODO check for failure of flush to disk - taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); + int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); + if (count != pFuncInfo->codeSize) { + fnError("udfd write udf shared library failed"); + msgInfo->code = TSDB_CODE_FILE_CORRUPTED; + } taosCloseFile(&file); strncpy(udf->path, path, strlen(path)); tFreeSFuncInfo(pFuncInfo); @@ -265,235 +524,88 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { char *startSuffix = "_start"; strncpy(startFuncName, processFuncName, strlen(processFuncName)); strncat(startFuncName, startSuffix, strlen(startSuffix)); - uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggStartFunc)); - char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; - char *finishSuffix = "_finish"; - strncpy(finishFuncName, processFuncName, strlen(processFuncName)); - strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); - uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); - // TODO: merge - } - return 0; -} - -void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { - // TODO: tracable id from client. connect, setup, call, teardown - fnInfo("setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName); - SUdfSetupRequest *setup = &request->setup; - int32_t code = TSDB_CODE_SUCCESS; - SUdf * udf = NULL; - uv_mutex_lock(&global.udfsMutex); - SUdf **udfInHash = taosHashGet(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName)); - if (udfInHash) { - ++(*udfInHash)->refCount; - udf = *udfInHash; - uv_mutex_unlock(&global.udfsMutex); - } else { - SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); - udfNew->refCount = 1; - udfNew->state = UDF_STATE_INIT; - - uv_mutex_init(&udfNew->lock); - uv_cond_init(&udfNew->condReady); - udf = udfNew; - taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), &udfNew, sizeof(&udfNew)); - uv_mutex_unlock(&global.udfsMutex); - } - - uv_mutex_lock(&udf->lock); - if (udf->state == UDF_STATE_INIT) { - udf->state = UDF_STATE_LOADING; - code = udfdLoadUdf(setup->udfName, udf); - if (udf->initFunc) { - udf->initFunc(); - } - udf->state = UDF_STATE_READY; - uv_cond_broadcast(&udf->condReady); - uv_mutex_unlock(&udf->lock); - } else { - while (udf->state != UDF_STATE_READY) { - uv_cond_wait(&udf->condReady, &udf->lock); - } - uv_mutex_unlock(&udf->lock); - } - SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); - handle->udf = udf; - - SUdfResponse rsp; - rsp.seqNum = request->seqNum; - rsp.type = request->type; - rsp.code = code; - rsp.setupRsp.udfHandle = (int64_t)(handle); - rsp.setupRsp.outputType = udf->outputType; - rsp.setupRsp.outputLen = udf->outputLen; - rsp.setupRsp.bufSize = udf->bufSize; - - int32_t len = encodeUdfResponse(NULL, &rsp); - rsp.msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, &rsp); - - uvUdf->output = uv_buf_init(bufBegin, len); - - taosMemoryFree(uvUdf->input.base); - return; -} - -void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { - SUdfCallRequest *call = &request->call; - fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request->seqNum, call->callType, call->udfHandle); - SUdfcFuncHandle * handle = (SUdfcFuncHandle *)(call->udfHandle); - SUdf * udf = handle->udf; - SUdfResponse response = {0}; - SUdfResponse * rsp = &response; - SUdfCallResponse *subRsp = &rsp->callRsp; - - int32_t code = TSDB_CODE_SUCCESS; - switch (call->callType) { - case TSDB_UDF_CALL_SCALA_PROC: { - SUdfColumn output = {0}; - - SUdfDataBlock input = {0}; - convertDataBlockToUdfDataBlock(&call->block, &input); - code = udf->scalarProcFunc(&input, &output); - freeUdfDataDataBlock(&input); - convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); - freeUdfColumn(&output); - break; - } - case TSDB_UDF_CALL_AGG_INIT: { - SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - udf->aggStartFunc(&outBuf); - subRsp->resultBuf = outBuf; - break; - } - case TSDB_UDF_CALL_AGG_PROC: { - SUdfDataBlock input = {0}; - convertDataBlockToUdfDataBlock(&call->block, &input); - SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - code = udf->aggProcFunc(&input, &call->interBuf, &outBuf); - freeUdfInterBuf(&call->interBuf); - freeUdfDataDataBlock(&input); - subRsp->resultBuf = outBuf; - - break; - } - case TSDB_UDF_CALL_AGG_FIN: { - SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - code = udf->aggFinishFunc(&call->interBuf, &outBuf); - freeUdfInterBuf(&call->interBuf); - subRsp->resultBuf = outBuf; - break; - } - default: - break; + uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggStartFunc)); + char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; + char *finishSuffix = "_finish"; + strncpy(finishFuncName, processFuncName, strlen(processFuncName)); + strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); + uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); + // TODO: merge + } + return 0; +} +static bool udfdRpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT) { + return true; + } else { + return false; } +} - rsp->seqNum = request->seqNum; - rsp->type = request->type; - rsp->code = code; - subRsp->callType = call->callType; +int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) { + pEpSet->version = 0; - int32_t len = encodeUdfResponse(NULL, rsp); - rsp->msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, rsp); - uvUdf->output = uv_buf_init(bufBegin, len); + // init mnode ip set + SEpSet *mgmtEpSet = &(pEpSet->epSet); + mgmtEpSet->numOfEps = 0; + mgmtEpSet->inUse = 0; - switch (call->callType) { - case TSDB_UDF_CALL_SCALA_PROC: { - tDeleteSSDataBlock(&call->block); - tDeleteSSDataBlock(&subRsp->resultData); - break; - } - case TSDB_UDF_CALL_AGG_INIT: { - freeUdfInterBuf(&subRsp->resultBuf); - break; - } - case TSDB_UDF_CALL_AGG_PROC: { - tDeleteSSDataBlock(&call->block); - freeUdfInterBuf(&subRsp->resultBuf); - break; - } - case TSDB_UDF_CALL_AGG_FIN: { - freeUdfInterBuf(&subRsp->resultBuf); - break; + if (firstEp && firstEp[0] != 0) { + if (strlen(firstEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; } - default: - break; - } - - taosMemoryFree(uvUdf->input.base); - return; -} -void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { - SUdfTeardownRequest *teardown = &request->teardown; - fnInfo("teardown. seq number: %" PRId64 ", handle:%" PRIx64, request->seqNum, teardown->udfHandle); - SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); - SUdf * udf = handle->udf; - bool unloadUdf = false; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return terrno; + } - uv_mutex_lock(&global.udfsMutex); - udf->refCount--; - if (udf->refCount == 0) { - unloadUdf = true; - taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); + mgmtEpSet->numOfEps++; } - uv_mutex_unlock(&global.udfsMutex); - if (unloadUdf) { - uv_cond_destroy(&udf->condReady); - uv_mutex_destroy(&udf->lock); - if (udf->destroyFunc) { - (udf->destroyFunc)(); + + if (secondEp && secondEp[0] != 0) { + if (strlen(secondEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; } - uv_dlclose(&udf->lib); - taosMemoryFree(udf); + + taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); + mgmtEpSet->numOfEps++; } - taosMemoryFree(handle); - SUdfResponse response; - SUdfResponse *rsp = &response; - rsp->seqNum = request->seqNum; - rsp->type = request->type; - rsp->code = code; - int32_t len = encodeUdfResponse(NULL, rsp); - rsp->msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, rsp); - uvUdf->output = uv_buf_init(bufBegin, len); + if (mgmtEpSet->numOfEps == 0) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } - taosMemoryFree(uvUdf->input.base); - return; + return 0; } -void udfdProcessRequest(uv_work_t *req) { - SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); - SUdfRequest request = {0}; - decodeUdfRequest(uvUdf->input.base, &request); - - switch (request.type) { - case UDF_TASK_SETUP: { - udfdProcessSetupRequest(uvUdf, &request); - break; - } +int32_t udfdOpenClientRpc() { + SRpcInit rpcInit = {0}; + rpcInit.label = "UDFD"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = (RpcCfp)udfdProcessRpcRsp; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = TSDB_DEFAULT_USER; + rpcInit.parent = &global; + rpcInit.rfp = udfdRpcRfp; - case UDF_TASK_CALL: { - udfdProcessCallRequest(uvUdf, &request); - break; - } - case UDF_TASK_TEARDOWN: { - udfdProcessTeardownRequest(uvUdf, &request); - break; - } - default: { - break; - } + global.clientRpc = rpcOpen(&rpcInit); + if (global.clientRpc == NULL) { + fnError("failed to init dnode rpc client"); + return -1; } + return 0; +} + +int32_t udfdCloseClientRpc() { + rpcClose(global.clientRpc); + return 0; } void udfdOnWrite(uv_write_t *req, int status) { @@ -529,7 +641,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->base = ctx->inputBuf; buf->len = ctx->inputCap; } else { - // TODO: log error + fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } @@ -541,7 +653,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->base = ctx->inputBuf + ctx->inputLen; buf->len = ctx->inputCap - ctx->inputLen; } else { - // TODO: log error + fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } @@ -580,8 +692,6 @@ void udfdPipeCloseCb(uv_handle_t *pipe) { taosMemoryFree(conn); } -void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn->client, udfdPipeCloseCb); } - void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { fnDebug("udf read %zd bytes from client", nread); if (nread == 0) return; @@ -599,7 +709,7 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } if (nread < 0) { - fnDebug("Receive error %s", uv_err_name(nread)); + fnError("Receive error %s", uv_err_name(nread)); if (nread == UV_EOF) { // TODO check more when close } else { @@ -638,91 +748,6 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { uv_stop(global.loop); } -static bool udfdRpcRfp(int32_t code) { - if (code == TSDB_CODE_RPC_REDIRECT) { - return true; - } else { - return false; - } -} - -int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) { - pEpSet->version = 0; - - // init mnode ip set - SEpSet *mgmtEpSet = &(pEpSet->epSet); - mgmtEpSet->numOfEps = 0; - mgmtEpSet->inUse = 0; - - if (firstEp && firstEp[0] != 0) { - if (strlen(firstEp) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); - if (code != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return terrno; - } - - mgmtEpSet->numOfEps++; - } - - if (secondEp && secondEp[0] != 0) { - if (strlen(secondEp) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); - mgmtEpSet->numOfEps++; - } - - if (mgmtEpSet->numOfEps == 0) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - return 0; -} - -int32_t udfdOpenClientRpc() { - SRpcInit rpcInit = {0}; - rpcInit.label = "UDFD"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = (RpcCfp)udfdProcessRpcRsp; - rpcInit.sessions = 1024; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = TSDB_DEFAULT_USER; - rpcInit.parent = &global; - rpcInit.rfp = udfdRpcRfp; - - global.clientRpc = rpcOpen(&rpcInit); - if (global.clientRpc == NULL) { - fnError("failed to init dnode rpc client"); - return -1; - } - return 0; -} - -int32_t udfdCloseClientRpc() { - rpcClose(global.clientRpc); - return 0; -} - -static void udfdPrintVersion() { -#ifdef TD_ENTERPRISE - char *releaseName = "enterprise"; -#else - char *releaseName = "community"; -#endif - printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); - printf("buildInfo: %s\n", buildinfo); -} - static int32_t udfdParseArgs(int32_t argc, char *argv[]) { for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { @@ -745,6 +770,17 @@ static int32_t udfdParseArgs(int32_t argc, char *argv[]) { return 0; } +static void udfdPrintVersion() { +#ifdef TD_ENTERPRISE + char *releaseName = "enterprise"; +#else + char *releaseName = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("buildInfo: %s\n", buildinfo); +} + static int32_t udfdInitLog() { char logName[12] = {0}; snprintf(logName, sizeof(logName), "%slog", "udfd"); @@ -868,8 +904,8 @@ int main(int argc, char *argv[]) { int32_t retryMnodeTimes = 0; int32_t code = 0; - while (retryMnodeTimes++ < TSDB_MAX_REPLICA) { - uv_sleep(500 * (1 << retryMnodeTimes)); + while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) { + uv_sleep(100 * (1 << retryMnodeTimes)); code = udfdConnectToMnode(); if (code == 0) { break; @@ -890,6 +926,7 @@ int main(int argc, char *argv[]) { udfdRun(); removeListeningPipe(); - udfdCloseClientRpc(); + + return 0; } diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 7dc66e47898eed6e58288d775a0d4c78f79dc4b0..d5fd574aadaf2e0c228b6f0dfd54fd809b00ac0f 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -31,7 +31,7 @@ if (${BUILD_WITH_INVERTEDINDEX}) endif(${BUILD_WITH_INVERTEDINDEX}) -if (${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) +#if (${BUILD_TEST}) +# add_subdirectory(test) +#endif(${BUILD_TEST}) diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index d474d874090b09a4a9952e25a8a8f84eca77bf10..aff2e0e836c0f2aae9a1fe63dd984cd4f5eb7850 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -63,7 +63,10 @@ typedef struct CacheTerm { IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type); +void indexCacheForceToMerge(void* cache); void indexCacheDestroy(void* cache); +void indexCacheBroadcast(void* cache); +void indexCacheWait(void* cache); Iterate* indexCacheIteratorCreate(IndexCache* cache); void indexCacheIteratorDestroy(Iterate* iiter); diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 27c380beafb53065f7bfd4e5955be234406a58f6..0bdcb131b69befd518b233e38a2653a17e67bde8 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -58,6 +58,8 @@ struct SIndex { SIndexStat stat; TdThreadMutex mtx; + tsem_t sem; + bool quit; }; struct SIndexOpts { @@ -69,6 +71,7 @@ struct SIndexOpts { int32_t cacheSize; // MB // add cache module later #endif + int32_t cacheOpt; // MB }; struct SIndexMultiTermQuery { @@ -131,42 +134,14 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); -#define indexFatal(...) \ - do { \ - if (sDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("index FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ - } \ - } while (0) -#define indexError(...) \ - do { \ - if (sDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("index ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ - } \ - } while (0) -#define indexWarn(...) \ - do { \ - if (sDebugFlag & DEBUG_WARN) { \ - taosPrintLog("index WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ - } \ - } while (0) -#define indexInfo(...) \ - do { \ - if (sDebugFlag & DEBUG_INFO) { \ - taosPrintLog("index ", DEBUG_INFO, 255, __VA_ARGS__); \ - } \ - } while (0) -#define indexDebug(...) \ - do { \ - if (sDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("index ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \ - } \ - } while (0) -#define indexTrace(...) \ - do { \ - if (sDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("index ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \ - } \ - } while (0) +// clang-format off +#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("INDEX FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while (0) +#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("INDEX ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while (0) +#define indexWarn(...) do { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("INDEX WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while (0) +#define indexInfo(...) do { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("INDEX ", DEBUG_INFO, 255, __VA_ARGS__); } } while (0) +#define indexDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("INDEX ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__);} } while (0) +#define indexTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("INDEX ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__);} } while (0) +// clang-format on #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 162d64c41c472e88aa019b4b5def7727850c636d..2141e90bbd4678e588e354d30be12211c7ada6df 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -90,6 +90,15 @@ static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, Iterat // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); +static void indexPost(void* idx) { + SIndex* pIdx = idx; + tsem_post(&pIdx->sem); +} +static void indexWait(void* idx) { + SIndex* pIdx = idx; + tsem_wait(&pIdx->sem); +} + int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { taosThreadOnce(&isInit, indexInit); SIndex* sIdx = taosMemoryCalloc(1, sizeof(SIndex)); @@ -107,6 +116,8 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { sIdx->cVersion = 1; sIdx->path = tstrdup(path); taosThreadMutexInit(&sIdx->mtx, NULL); + tsem_init(&sIdx->sem, 0, 0); + // taosThreadCondInit(&sIdx->finished, NULL); sIdx->refId = indexAddRef(sIdx); indexAcquireRef(sIdx->refId); @@ -124,16 +135,8 @@ END: void indexDestroy(void* handle) { SIndex* sIdx = handle; - void* iter = taosHashIterate(sIdx->colObj, NULL); - while (iter) { - IndexCache** pCache = iter; - if (*pCache) { - indexCacheUnRef(*pCache); - } - iter = taosHashIterate(sIdx->colObj, iter); - } - taosHashCleanup(sIdx->colObj); taosThreadMutexDestroy(&sIdx->mtx); + tsem_destroy(&sIdx->sem); indexTFileDestroy(sIdx->tindex); taosMemoryFree(sIdx->path); taosMemoryFree(sIdx); @@ -141,6 +144,20 @@ void indexDestroy(void* handle) { } void indexClose(SIndex* sIdx) { indexReleaseRef(sIdx->refId); + bool ref = 0; + if (sIdx->colObj != NULL) { + void* iter = taosHashIterate(sIdx->colObj, NULL); + while (iter) { + IndexCache** pCache = iter; + indexCacheForceToMerge((void*)(*pCache)); + indexWait((void*)(sIdx)); + iter = taosHashIterate(sIdx->colObj, iter); + indexCacheUnRef(*pCache); + } + taosHashCleanup(sIdx->colObj); + sIdx->colObj = NULL; + } + // taosMsleep(1000 * 5); indexRemoveRef(sIdx->refId); } int64_t indexAddRef(void* p) { @@ -451,6 +468,18 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { } // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); + if (cacheIter == NULL) { + indexError("%p immtable is empty, ignore merge opera", pCache); + indexCacheDestroyImm(pCache); + tfileReaderUnRef(pReader); + if (sIdx->quit) { + indexPost(sIdx); + // indexCacheBroadcast(pCache); + } + indexReleaseRef(sIdx->refId); + return 0; + } + Iterate* tfileIter = tfileIteratorCreate(pReader); if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); @@ -506,7 +535,11 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { } else { indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000); } + if (sIdx->quit) { + indexPost(sIdx); + } indexReleaseRef(sIdx->refId); + return ret; } void iterateValueDestroy(IterateValue* value, bool destroy) { @@ -563,10 +596,11 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON); char* p = buf; - SERIALIZE_MEM_TO_BUF(buf, key, suid); + char tbuf[65] = {0}; + indexInt2str((int64_t)key->suid, tbuf, 0); + + SERIALIZE_STR_VAR_TO_BUF(buf, tbuf, strlen(tbuf)); SERIALIZE_VAR_TO_BUF(buf, '_', char); - // SERIALIZE_MEM_TO_BUF(buf, key, colType); - // SERIALIZE_VAR_TO_BUF(buf, '_', char); if (hasJson) { SERIALIZE_STR_VAR_TO_BUF(buf, JSON_COLUMN, strlen(JSON_COLUMN)); } else { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 9a2e487df1f5880dba5472574199e0bdcfbb58be..232eca9304c0168a73d41298cb01d626e29302c9 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -23,6 +23,7 @@ #define MEM_TERM_LIMIT 10 * 10000 #define MEM_THRESHOLD 64 * 1024 +#define MEM_SIGNAL_QUIT MEM_THRESHOLD * 20 #define MEM_ESTIMATE_RADIO 1.5 static void indexMemRef(MemTable* tbl); @@ -385,7 +386,7 @@ void indexCacheDebug(IndexCache* cache) { void indexCacheDestroySkiplist(SSkipList* slt) { SSkipListIterator* iter = tSkipListCreateIter(slt); - while (tSkipListIterNext(iter)) { + while (iter != NULL && tSkipListIterNext(iter)) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); if (ct != NULL) { @@ -396,17 +397,24 @@ void indexCacheDestroySkiplist(SSkipList* slt) { tSkipListDestroyIter(iter); tSkipListDestroy(slt); } +void indexCacheBroadcast(void* cache) { + IndexCache* pCache = cache; + taosThreadCondBroadcast(&pCache->finished); +} +void indexCacheWait(void* cache) { + IndexCache* pCache = cache; + taosThreadCondWait(&pCache->finished, &pCache->mtx); +} void indexCacheDestroyImm(IndexCache* cache) { if (cache == NULL) { return; } - MemTable* tbl = NULL; taosThreadMutexLock(&cache->mtx); tbl = cache->imm; cache->imm = NULL; // or throw int bg thread - taosThreadCondBroadcast(&cache->finished); + indexCacheBroadcast(cache); taosThreadMutexUnlock(&cache->mtx); @@ -429,11 +437,13 @@ void indexCacheDestroy(void* cache) { } Iterate* indexCacheIteratorCreate(IndexCache* cache) { + if (cache->imm == NULL) { + return NULL; + } Iterate* iiter = taosMemoryCalloc(1, sizeof(Iterate)); if (iiter == NULL) { return NULL; } - taosThreadMutexLock(&cache->mtx); indexMemRef(cache->imm); @@ -458,17 +468,16 @@ void indexCacheIteratorDestroy(Iterate* iter) { taosMemoryFree(iter); } -int indexCacheSchedToMerge(IndexCache* pCache) { +int indexCacheSchedToMerge(IndexCache* pCache, bool notify) { SSchedMsg schedMsg = {0}; schedMsg.fp = doMergeWork; schedMsg.ahandle = pCache; - schedMsg.thandle = NULL; - // schedMsg.thandle = taosMemoryCalloc(1, sizeof(int64_t)); - // memcpy((char*)(schedMsg.thandle), (char*)&(pCache->index->refId), sizeof(int64_t)); + if (notify) { + schedMsg.thandle = taosMemoryMalloc(1); + } schedMsg.msg = NULL; indexAcquireRef(pCache->index->refId); taosScheduleTask(indexQhandle, &schedMsg); - return 0; } @@ -478,8 +487,10 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - taosThreadCondWait(&cache->finished, &cache->mtx); + indexCacheWait(cache); } else { + bool notifyQuit = cache->occupiedMem >= MEM_SIGNAL_QUIT ? true : false; + indexCacheRef(cache); cache->imm = cache->mem; cache->mem = indexInternalCacheCreate(cache->type); @@ -487,7 +498,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { cache->occupiedMem = 0; // sched to merge // unref cache in bgwork - indexCacheSchedToMerge(cache); + indexCacheSchedToMerge(cache, notifyQuit); } } } @@ -533,6 +544,19 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { return 0; // encode end } +void indexCacheForceToMerge(void* cache) { + IndexCache* pCache = cache; + indexCacheRef(pCache); + taosThreadMutexLock(&pCache->mtx); + + indexInfo("%p is forced to merge into tfile", pCache); + pCache->occupiedMem += MEM_SIGNAL_QUIT; + indexCacheMakeRoomForWrite(pCache); + + taosThreadMutexUnlock(&pCache->mtx); + indexCacheUnRef(pCache); + return; +} int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { IndexCache* pCache = cache; return 0; @@ -691,6 +715,9 @@ static MemTable* indexInternalCacheCreate(int8_t type) { static void doMergeWork(SSchedMsg* msg) { IndexCache* pCache = msg->ahandle; SIndex* sidx = (SIndex*)pCache->index; + + sidx->quit = msg->thandle ? true : false; + taosMemoryFree(msg->thandle); indexFlushCacheToTFile(sidx, pCache); } static bool indexCacheIteratorNext(Iterate* itera) { @@ -709,9 +736,6 @@ static bool indexCacheIteratorNext(Iterate* itera) { iv->type = ct->operaType; iv->ver = ct->version; iv->colVal = tstrdup(ct->colVal); - // printf("col Val: %s\n", iv->colVal); - // iv->colType = cv->colType; - taosArrayPush(iv->val, &ct->uid); } return next; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 4c23e4ba4b9a89d124a93a434da27158891816bc..78c7babb681e44629281f0ffd6ea6ba835495b5b 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -22,6 +22,29 @@ #include "ttypes.h" #include "tvariant.h" +#define INDEX_DATA_BOOL_NULL 0x02 +#define INDEX_DATA_TINYINT_NULL 0x80 +#define INDEX_DATA_SMALLINT_NULL 0x8000 +#define INDEX_DATA_INT_NULL 0x80000000L +#define INDEX_DATA_BIGINT_NULL 0x8000000000000000L +#define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL + +#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF +#define INDEX_DATA_BINARY_NULL 0xFF +#define INDEX_DATA_JSON_NULL 0xFFFFFFFF +#define INDEX_DATA_JSON_null 0xFFFFFFFE +#define INDEX_DATA_JSON_NOT_NULL 0x01 + +#define INDEX_DATA_UTINYINT_NULL 0xFF +#define INDEX_DATA_USMALLINT_NULL 0xFFFF +#define INDEX_DATA_UINT_NULL 0xFFFFFFFF +#define INDEX_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL + +#define INDEX_DATA_NULL_STR "NULL" +#define INDEX_DATA_NULL_STR_L "null" + char JSON_COLUMN[] = "JSON"; char JSON_VALUE_DELIM = '&'; @@ -372,7 +395,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); - *dst = (char*) * dst - tlen; + *dst = (char*)*dst - tlen; break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index dd6117ed2ac9aa7c0add1c1e5015543187877942..f5b3cbf2271112687e902fda1cb221cc45521563 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -141,7 +141,6 @@ void tfileCacheDestroy(TFileCache* tcache) { TFileReader* p = *reader; indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType); - tfileReaderUnRef(p); reader = taosHashIterate(tcache->tableCache, reader); } @@ -172,7 +171,6 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { oldReader->remove = true; tfileReaderUnRef(oldReader); } - taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); tfileReaderRef(reader); return; @@ -500,15 +498,15 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) { SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; - + int ret = 0; if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { - return tfSearch[1][qtype](reader, term, tr); + ret = tfSearch[1][qtype](reader, term, tr); } else { - return tfSearch[0][qtype](reader, term, tr); + ret = tfSearch[0][qtype](reader, term, tr); } tfileReaderUnRef(reader); - return 0; + return ret; } TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) { diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 679e24f1a7eea48ef815b59c662d9212d755004c..a2d7adf1c7fc6e9ea2c1996206280f953939fd65 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -15,7 +15,7 @@ #include "tutil.h" void* callback(void* s) { return s; } -static std::string fileName = "/tmp/tindex.tindex"; +static std::string fileName = TD_TMP_DIR_PATH "tindex.tindex"; class FstWriter { public: FstWriter() { @@ -48,7 +48,7 @@ class FstWriter { class FstReadMemory { public: - FstReadMemory(int32_t size, const std::string& fileName = "/tmp/tindex.tindex") { + FstReadMemory(int32_t size, const std::string& fileName = TD_TMP_DIR_PATH "tindex.tindex") { _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024); _w = fstCountingWriterCreate(_wc); _size = size; diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index ab6c1a47045ea4957dcfd57ccda8c9bbcd8d7a90..136c4dafecdceba665d4a8657ed2053fa2486675 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -17,7 +17,7 @@ #include "tskiplist.h" #include "tutil.h" -static std::string dir = "/tmp/index"; +static std::string dir = TD_TMP_DIR_PATH "index"; static char indexlog[PATH_MAX] = {0}; static char tindex[PATH_MAX] = {0}; diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 733f1b4ed1f49a7c25a1f7d2c5be8466cd75bd15..11a25a798fb06f76c4597c193cac3dbf8710bea2 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -51,7 +51,7 @@ class DebugInfo { class FstWriter { public: FstWriter() { - _wc = writerCtxCreate(TFile, "/tmp/tindex", false, 64 * 1024 * 1024); + _wc = writerCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", false, 64 * 1024 * 1024); _b = fstBuilderCreate(NULL, 0); } bool Put(const std::string& key, uint64_t val) { @@ -75,7 +75,7 @@ class FstWriter { class FstReadMemory { public: FstReadMemory(size_t size) { - _wc = writerCtxCreate(TFile, "/tmp/tindex", true, 64 * 1024); + _wc = writerCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", true, 64 * 1024); _w = fstCountingWriterCreate(_wc); _size = size; memset((void*)&_s, 0, sizeof(_s)); @@ -272,9 +272,26 @@ void validateFst() { } delete m; } +static std::string logDir = "/tmp/log"; + +static void initLog() { + const char* defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + sDebugFlag = 143; + strcpy(tsLogDir, logDir.c_str()); + taosRemoveDir(tsLogDir); + taosMkDir(tsLogDir); + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} class IndexEnv : public ::testing::Test { protected: virtual void SetUp() { + initLog(); taosRemoveDir(path); opts = indexOptsCreate(); int ret = indexOpen(opts, path, &index); @@ -285,7 +302,7 @@ class IndexEnv : public ::testing::Test { indexOptsDestroy(opts); } - const char* path = "/tmp/tindex"; + const char* path = TD_TMP_DIR_PATH "tindex"; SIndexOpts* opts; SIndex* index; }; @@ -342,7 +359,7 @@ class IndexEnv : public ::testing::Test { class TFileObj { public: - TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage") + TFileObj(const std::string& path = TD_TMP_DIR_PATH "tindex", const std::string& colName = "voltage") : path_(path), colName_(colName) { colId_ = 10; reader_ = NULL; @@ -437,7 +454,7 @@ class IndexTFileEnv : public ::testing::Test { // tfileWriterDestroy(twrite); } TFileObj* fObj; - std::string dir = "/tmp/tindex"; + std::string dir = TD_TMP_DIR_PATH "tindex"; std::string colName = "voltage"; int coldId = 2; @@ -804,7 +821,7 @@ class IndexObj { } ~IndexObj() { - indexCleanUp(); + // indexCleanUp(); indexClose(idx); } @@ -817,12 +834,15 @@ class IndexObj { class IndexEnv2 : public ::testing::Test { protected: - virtual void SetUp() { index = new IndexObj(); } + virtual void SetUp() { + initLog(); + index = new IndexObj(); + } virtual void TearDown() { delete index; } IndexObj* index; }; TEST_F(IndexEnv2, testIndexOpen) { - std::string path = "/tmp/test"; + std::string path = TD_TMP_DIR_PATH "test"; if (index->Init(path) != 0) { std::cout << "failed to init index" << std::endl; exit(1); @@ -884,14 +904,37 @@ TEST_F(IndexEnv2, testIndexOpen) { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); index->Search(mq, result); std::cout << "target size: " << taosArrayGetSize(result) << std::endl; - assert(taosArrayGetSize(result) == 400); + EXPECT_EQ(400, taosArrayGetSize(result)); taosArrayDestroy(result); indexMultiTermQueryDestroy(mq); } } +TEST_F(IndexEnv2, testEmptyIndexOpen) { + std::string path = "/tmp/test"; + if (index->Init(path) != 0) { + std::cout << "failed to init index" << std::endl; + exit(1); + } + + int targetSize = 1; + { + std::string colName("tag1"), colVal("Hello"); + + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < targetSize; i++) { + int tableId = i; + int ret = index->Put(terms, tableId); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + } +} TEST_F(IndexEnv2, testIndex_TrigeFlush) { - std::string path = "/tmp/testxxx"; + std::string path = TD_TMP_DIR_PATH "testxxx"; if (index->Init(path) != 0) { // r std::cout << "failed to init" << std::endl; @@ -914,7 +957,7 @@ static void multi_write_and_search(IndexObj* idx) { idx->WriteMultiMillonData("tag2", "world test nothing", 100 * 10); } TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { // opt } @@ -934,7 +977,7 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { } } TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } @@ -950,14 +993,14 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { } // TEST_F(IndexEnv2, testIndex_restart) { -// std::string path = "/tmp/cache_and_tfile"; +// std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; // if (index->Init(path) != 0) { // } // index->SearchOneTarget("tag1", "Hello", 10); // index->SearchOneTarget("tag2", "Test", 10); //} // TEST_F(IndexEnv2, testIndex_restart1) { -// std::string path = "/tmp/cache_and_tfile"; +// std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; // if (index->Init(path) != 0) { // } // index->ReadMultiMillonData("tag1", "coding"); @@ -966,7 +1009,7 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { //} // TEST_F(IndexEnv2, testIndex_read_performance) { -// std::string path = "/tmp/cache_and_tfile"; +// std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; // if (index->Init(path) != 0) { // } // index->PutOneTarge("tag1", "Hello", 12); @@ -976,7 +1019,7 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { // assert(3 == index->SearchOne("tag1", "Hello")); //} // TEST_F(IndexEnv2, testIndexMultiTag) { -// std::string path = "/tmp/multi_tag"; +// std::string path = TD_TMP_DIR_PATH "multi_tag"; // if (index->Init(path) != 0) { // } // int64_t st = taosGetTimestampUs(); @@ -986,7 +1029,7 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { // // index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000); //} TEST_F(IndexEnv2, testLongComVal1) { - std::string path = "/tmp/long_colVal"; + std::string path = TD_TMP_DIR_PATH "long_colVal"; if (index->Init(path) != 0) { } // gen colVal by randstr @@ -995,7 +1038,7 @@ TEST_F(IndexEnv2, testLongComVal1) { } TEST_F(IndexEnv2, testLongComVal2) { - std::string path = "/tmp/long_colVal"; + std::string path = TD_TMP_DIR_PATH "long_colVal"; if (index->Init(path) != 0) { } // gen colVal by randstr @@ -1003,7 +1046,7 @@ TEST_F(IndexEnv2, testLongComVal2) { index->WriteMultiMillonData("tag1", randstr, 100 * 1000); } TEST_F(IndexEnv2, testLongComVal3) { - std::string path = "/tmp/long_colVal"; + std::string path = TD_TMP_DIR_PATH "long_colVal"; if (index->Init(path) != 0) { } // gen colVal by randstr @@ -1011,7 +1054,7 @@ TEST_F(IndexEnv2, testLongComVal3) { index->WriteMultiMillonData("tag1", randstr, 100 * 1000); } TEST_F(IndexEnv2, testLongComVal4) { - std::string path = "/tmp/long_colVal"; + std::string path = TD_TMP_DIR_PATH "long_colVal"; if (index->Init(path) != 0) { } // gen colVal by randstr @@ -1019,7 +1062,7 @@ TEST_F(IndexEnv2, testLongComVal4) { index->WriteMultiMillonData("tag1", randstr, 100 * 100); } TEST_F(IndexEnv2, testIndex_read_performance1) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } index->PutOneTarge("tag1", "Hello", 12); @@ -1029,7 +1072,7 @@ TEST_F(IndexEnv2, testIndex_read_performance1) { EXPECT_EQ(2, index->SearchOne("tag1", "Hello")); } TEST_F(IndexEnv2, testIndex_read_performance2) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } index->PutOneTarge("tag1", "Hello", 12); @@ -1039,7 +1082,7 @@ TEST_F(IndexEnv2, testIndex_read_performance2) { EXPECT_EQ(2, index->SearchOne("tag1", "Hello")); } TEST_F(IndexEnv2, testIndex_read_performance3) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } index->PutOneTarge("tag1", "Hello", 12); @@ -1049,7 +1092,7 @@ TEST_F(IndexEnv2, testIndex_read_performance3) { EXPECT_EQ(2, index->SearchOne("tag1", "Hello")); } TEST_F(IndexEnv2, testIndex_read_performance4) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } index->PutOneTarge("tag10", "Hello", 12); @@ -1059,7 +1102,7 @@ TEST_F(IndexEnv2, testIndex_read_performance4) { EXPECT_EQ(1, index->SearchOne("tag10", "Hello")); } TEST_F(IndexEnv2, testIndex_cache_del) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } for (int i = 0; i < 100; i++) { @@ -1098,7 +1141,7 @@ TEST_F(IndexEnv2, testIndex_cache_del) { } TEST_F(IndexEnv2, testIndex_del) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } for (int i = 0; i < 100; i++) { diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index e827d1763f2b9e505118f6d0b61a26e82f83aa55..8a837c5700da2b8c70d083d5f282933844091673 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -16,8 +16,8 @@ #include "tskiplist.h" #include "tutil.h" -static std::string dir = "/tmp/json"; -static std::string logDir = "/tmp/log"; +static std::string dir = TD_TMP_DIR_PATH "json"; +static std::string logDir = TD_TMP_DIR_PATH "log"; static void initLog() { const char* defaultLogFileNamePrefix = "taoslog"; diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 10bb47d3ffd9ed59b72aab3d528b19d909bf4e8c..2cba1eb043e0e4063b4a7519a3c116820412c69d 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -241,7 +241,7 @@ alter_table_clause(A) ::= alter_table_clause(A) ::= full_table_name(B) RENAME TAG column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &C, &D); } alter_table_clause(A) ::= - full_table_name(B) SET TAG column_name(C) NK_EQ literal(D). { A = createAlterTableSetTag(pCxt, B, &C, releaseRawExprNode(pCxt, D)); } + full_table_name(B) SET TAG column_name(C) NK_EQ signed_literal(D). { A = createAlterTableSetTag(pCxt, B, &C, D); } %type multi_create_clause { SNodeList* } %destructor multi_create_clause { nodesDestroyList($$); } @@ -448,7 +448,7 @@ agg_func_opt(A) ::= AGGREGATE. %type bufsize_opt { int32_t } %destructor bufsize_opt { } bufsize_opt(A) ::= . { A = 0; } -bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). { A = strtol(B.z, NULL, 10); } +bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). { A = taosStr2Int32(B.z, NULL, 10); } /************************************************ create/drop stream **************************************************/ cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index b4529506245b1b58b10959322595053382b228ee..11324e3f49af1027fb6244bf49d78a14b427e782 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -827,7 +827,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); if (NULL == row) { - return buildInvalidOperationMsg(&pCxt->msg, "tag value expected"); + return buildInvalidOperationMsg(&pCxt->msg, "out of memory"); } tdSortKVRowByColIdx(row); @@ -1085,6 +1085,10 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // no data in the sql string anymore. if (sToken.n == 0) { + if (sToken.type && pCxt->pSql[0]) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z); + } + if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); } @@ -1347,7 +1351,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN SKVRow row = tdGetKVRowFromBuilder(&tagBuilder); if (NULL == row) { tdDestroyKVRowBuilder(&tagBuilder); - return buildInvalidOperationMsg(&pBuf, "tag value expected"); + return buildInvalidOperationMsg(&pBuf, "out of memory"); } tdSortKVRowByColIdx(row); @@ -1696,7 +1700,7 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD *row = tdGetKVRowFromBuilder(tagsBuilder); if (*row == NULL) { - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_OUT_OF_MEMORY; } tdSortKVRowByColIdx(*row); return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index abc7ddb17f4b6ea4cd9a859637fe3886dffa21fc..8fb9780f8a5b52c62822c25eb1b52be40d30c1d9 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -704,6 +704,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) { if (t0.type == TK_NK_SEMI) { t0.n = 0; + t0.type = 0; return t0; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f6d53dd15ac5f4c1e3b9f1d1e07f5dc34ab2ff40..99e1135599a86961ad9794dad32da6fe627ec2c3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -694,66 +694,110 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { return translateValueImpl(pCxt, pVal, pVal->node.resType); } -static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { - if (nodesIsUnaryOp(pOp)) { - if (OP_TYPE_MINUS == pOp->opType) { - if (!IS_MATHABLE_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); - } - pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; - } else { - pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; +static bool isMultiResFunc(SNode* pNode) { + if (NULL == pNode) { + return false; + } + if (QUERY_NODE_FUNCTION != nodeType(pNode) || !fmIsMultiResFunc(((SFunctionNode*)pNode)->funcId)) { + return false; + } + SNodeList* pParameterList = ((SFunctionNode*)pNode)->pParameterList; + if (LIST_LENGTH(pParameterList) > 1) { + return true; + } + SNode* pParam = nodesListGetNode(pParameterList, 0); + return (QUERY_NODE_COLUMN == nodeType(pParam) ? 0 == strcmp(((SColumnNode*)pParam)->colName, "*") : false); +} + +static EDealRes translateUnaryOperator(STranslateContext* pCxt, SOperatorNode* pOp) { + if (OP_TYPE_MINUS == pOp->opType) { + if (!IS_MATHABLE_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); } - return DEAL_RES_CONTINUE; + pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + } else { + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; } + return DEAL_RES_CONTINUE; +} + +static EDealRes translateArithmeticOperator(STranslateContext* pCxt, SOperatorNode* pOp) { SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; - if (nodesIsArithmeticOp(pOp)) { - if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || - TSDB_DATA_TYPE_BLOB == rdt.type) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || + TSDB_DATA_TYPE_BLOB == rdt.type) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + } + if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || + (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && (IS_VAR_DATA_TYPE(rdt.type) || IS_FLOAT_TYPE(rdt.type))) || + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && (IS_VAR_DATA_TYPE(ldt.type) || IS_FLOAT_TYPE(ldt.type)))) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + } + + if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_INTEGER_TYPE(rdt.type)) || + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_INTEGER_TYPE(ldt.type)) || + (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_BOOL == rdt.type) || + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BOOL == ldt.type)) { + pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; + } else { + pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + } + return DEAL_RES_CONTINUE; +} + +static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNode* pOp) { + SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; + SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; + if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + } + if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { + ((SExprNode*)pOp->pRight)->resType = ((SExprNode*)pOp->pLeft)->resType; + } + if (nodesIsRegularOp(pOp)) { + if (!IS_STR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); } - if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || - (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && (IS_VAR_DATA_TYPE(rdt.type) || IS_FLOAT_TYPE(rdt.type))) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && (IS_VAR_DATA_TYPE(ldt.type) || IS_FLOAT_TYPE(ldt.type)))) { + if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || !IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } + } + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + return DEAL_RES_CONTINUE; +} - if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_INTEGER_TYPE(rdt.type)) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_INTEGER_TYPE(ldt.type)) || - (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_BOOL == rdt.type) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BOOL == ldt.type)) { - pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; - } else { - pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; - } +static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pOp) { + SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; + SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; + if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + } + pOp->node.resType.type = TSDB_DATA_TYPE_JSON; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes; + return DEAL_RES_CONTINUE; +} + +static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { + if (isMultiResFunc(pOp->pLeft)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + } + if (isMultiResFunc(pOp->pRight)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + } + + if (nodesIsUnaryOp(pOp)) { + return translateUnaryOperator(pCxt, pOp); + } else if (nodesIsArithmeticOp(pOp)) { + return translateArithmeticOperator(pCxt, pOp); } else if (nodesIsComparisonOp(pOp)) { - if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - } - if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { - ((SExprNode*)pOp->pRight)->resType = ((SExprNode*)pOp->pLeft)->resType; - } - if (nodesIsRegularOp(pOp)) { - if (!IS_STR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); - } - if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || !IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - } - } - pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + return translateComparisonOperator(pCxt, pOp); } else if (nodesIsJsonOp(pOp)) { - if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - } - pOp->node.resType.type = TSDB_DATA_TYPE_JSON; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes; + return translateJsonOperator(pCxt, pOp); } return DEAL_RES_CONTINUE; } @@ -808,6 +852,13 @@ static bool hasInvalidFuncNesting(SNodeList* pParameterList) { } static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { + SNode* pParam = NULL; + FOREACH(pParam, pFunc->pParameterList) { + if (isMultiResFunc(pParam)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pParam)->aliasName); + } + } + SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet, @@ -926,9 +977,10 @@ typedef struct SCheckExprForGroupByCxt { STranslateContext* pTranslateCxt; int32_t selectFuncNum; bool hasSelectValFunc; + bool hasOtherAggFunc; } SCheckExprForGroupByCxt; -static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, bool* pHasSelectValFunc, SNode** pNode) { +static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) { SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pFunc) { pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; @@ -942,9 +994,6 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, bool* pHasSel } if (TSDB_CODE_SUCCESS == pCxt->errCode) { *pNode = (SNode*)pFunc; - if (NULL != pHasSelectValFunc) { - *pHasSelectValFunc = true; - } } else { nodesDestroyNode(pFunc); } @@ -956,8 +1005,12 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) { return DEAL_RES_CONTINUE; } - pCxt->selectFuncNum += isSelectFunc(*pNode) ? 1 : 0; - if (pCxt->selectFuncNum > 1 && pCxt->hasSelectValFunc) { + if (isSelectFunc(*pNode)) { + ++(pCxt->selectFuncNum); + } else if (isAggFunc(*pNode)) { + pCxt->hasOtherAggFunc = true; + } + if ((pCxt->selectFuncNum > 1 && pCxt->hasSelectValFunc) || (pCxt->hasOtherAggFunc && pCxt->hasSelectValFunc)) { return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); } if (isAggFunc(*pNode) && !isDistinctOrderBy(pCxt->pTranslateCxt)) { @@ -970,10 +1023,11 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { - if (pCxt->selectFuncNum > 1) { + if (pCxt->selectFuncNum > 1 || pCxt->hasOtherAggFunc) { return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); } else { - return rewriteColToSelectValFunc(pCxt->pTranslateCxt, &pCxt->hasSelectValFunc, pNode); + pCxt->hasSelectValFunc = true; + return rewriteColToSelectValFunc(pCxt->pTranslateCxt, pNode); } } if (isAggFunc(*pNode) && isDistinctOrderBy(pCxt->pTranslateCxt)) { @@ -983,7 +1037,8 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode** pNode) { - SCheckExprForGroupByCxt cxt = {.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false}; + SCheckExprForGroupByCxt cxt = { + .pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false, .hasOtherAggFunc = false}; nodesRewriteExpr(pNode, doCheckExprForGroupBy, &cxt); if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); @@ -995,7 +1050,8 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList if (NULL == getGroupByList(pCxt)) { return TSDB_CODE_SUCCESS; } - SCheckExprForGroupByCxt cxt = {.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false}; + SCheckExprForGroupByCxt cxt = { + .pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false, .hasOtherAggFunc = false}; nodesRewriteExprs(pList, doCheckExprForGroupBy, &cxt); if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); @@ -1008,7 +1064,7 @@ static EDealRes rewriteColsToSelectValFuncImpl(SNode** pNode, void* pContext) { return DEAL_RES_IGNORE_CHILD; } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { - return rewriteColToSelectValFunc((STranslateContext*)pContext, NULL, pNode); + return rewriteColToSelectValFunc((STranslateContext*)pContext, pNode); } return DEAL_RES_CONTINUE; } @@ -1027,11 +1083,16 @@ typedef struct CheckAggColCoexistCxt { bool existCol; bool existNonstdFunc; int32_t selectFuncNum; + bool existOtherAggFunc; } CheckAggColCoexistCxt; static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; - pCxt->selectFuncNum += isSelectFunc(pNode) ? 1 : 0; + if (isSelectFunc(pNode)) { + ++(pCxt->selectFuncNum); + } else if (isAggFunc(pNode)) { + pCxt->existOtherAggFunc = true; + } if (isAggFunc(pNode)) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; @@ -1050,13 +1111,17 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (NULL != pSelect->pGroupByList) { return TSDB_CODE_SUCCESS; } - CheckAggColCoexistCxt cxt = { - .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false, .existNonstdFunc = false}; + CheckAggColCoexistCxt cxt = {.pTranslateCxt = pCxt, + .existAggFunc = false, + .existCol = false, + .existNonstdFunc = false, + .selectFuncNum = 0, + .existOtherAggFunc = false}; nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } - if (1 == cxt.selectFuncNum) { + if (1 == cxt.selectFuncNum && !cxt.existOtherAggFunc) { return rewriteColsToSelectValFunc(pCxt, pSelect); } if ((cxt.selectFuncNum > 1 || cxt.existAggFunc || NULL != pSelect->pWindow) && cxt.existCol) { @@ -1230,18 +1295,6 @@ static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) { return TSDB_CODE_SUCCESS; } -static bool isMultiResFunc(SNode* pNode) { - if (QUERY_NODE_FUNCTION != nodeType(pNode) || !fmIsMultiResFunc(((SFunctionNode*)pNode)->funcId)) { - return false; - } - SNodeList* pParameterList = ((SFunctionNode*)pNode)->pParameterList; - if (LIST_LENGTH(pParameterList) > 1) { - return true; - } - SNode* pParam = nodesListGetNode(pParameterList, 0); - return (QUERY_NODE_COLUMN == nodeType(pParam) ? 0 == strcmp(((SColumnNode*)pParam)->colName, "*") : false); -} - static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pFunc) { @@ -1872,7 +1925,7 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { } static bool dataTypeEqual(const SDataType* l, const SDataType* r) { - return (l->type == r->type && l->bytes == l->bytes && l->precision == r->precision && l->scale == l->scale); + return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale); } static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType dt, SNode** pCast) { @@ -2726,8 +2779,11 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm SName tableName; tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); - - return buildRollupAst(pCxt, pStmt, pReq); + int32_t code = collectUseTable(&tableName, pCxt->pTables); + if (TSDB_CODE_SUCCESS == code) { + code = buildRollupAst(pCxt, pStmt, pReq); + } + return code; } static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { @@ -4032,13 +4088,18 @@ static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* p return scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal); } +static int32_t colDataBytesToValueDataBytes(uint8_t type, int32_t bytes) { + if (TSDB_DATA_TYPE_VARCHAR == type || TSDB_DATA_TYPE_BINARY == type || TSDB_DATA_TYPE_VARBINARY == type) { + return bytes - VARSTR_HEADER_SIZE; + } else if (TSDB_DATA_TYPE_NCHAR == type) { + return (bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } + return bytes; +} + static SDataType schemaToDataType(SSchema* pSchema) { SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = 0, .scale = 0}; - if (TSDB_DATA_TYPE_VARCHAR == dt.type || TSDB_DATA_TYPE_BINARY == dt.type || TSDB_DATA_TYPE_VARBINARY == dt.type) { - dt.bytes -= VARSTR_HEADER_SIZE; - } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { - dt.bytes = (dt.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - } + dt.bytes = colDataBytesToValueDataBytes(pSchema->type, pSchema->bytes); return dt; } @@ -4393,8 +4454,10 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); pReq->nTagVal = pStmt->pVal->node.resType.bytes; - char* pVal = nodesGetValueFromNode(pStmt->pVal); - pReq->pTagVal = IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type) ? pVal + VARSTR_HEADER_SIZE : pVal; + if (TSDB_DATA_TYPE_NCHAR == pStmt->pVal->node.resType.type) { + pReq->nTagVal = pReq->nTagVal * TSDB_NCHAR_SIZE; + } + pReq->pTagVal = nodesGetValueFromNode(pStmt->pVal); return TSDB_CODE_SUCCESS; } @@ -4418,6 +4481,9 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, SVAlterTbReq* pReq) { + if (2 == getNumOfColumns(pTableMeta)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DROP_COL); + } SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); @@ -4440,7 +4506,8 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); - } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->bytes >= pReq->colModBytes) { + } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || + pSchema->bytes >= pReq->colModBytes) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index fe21915b1ae100948ab2b485d799456aafbda639..3df9c8abf7f36d5f04b9ce79070c71dd63a712ff 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -169,6 +169,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { "And, cannot be mixed with other non scalar functions or columns."; case TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY: return "Window query not supported, since the result of subquery not include valid timestamp column"; + case TSDB_CODE_PAR_INVALID_DROP_COL: + return "No columns can be dropped"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 2d844e18429fbaaed9e6a61cf3234c7dc5da26a3..0854bb83e471151e83efd066192ee576561b28ee 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -211,540 +211,538 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2167) +#define YY_ACTTAB_COUNT (2154) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 383, 76, 384, 1380, 391, 519, 384, 1380, 1631, 28, - /* 10 */ 223, 1467, 35, 33, 113, 1463, 343, 347, 1646, 1777, - /* 20 */ 301, 1470, 1159, 1627, 1635, 1633, 36, 34, 32, 31, - /* 30 */ 30, 1729, 1776, 1478, 94, 522, 1774, 93, 92, 91, - /* 40 */ 90, 89, 88, 87, 86, 85, 1662, 1157, 1523, 292, - /* 50 */ 32, 31, 30, 503, 291, 1726, 1777, 479, 14, 1521, - /* 60 */ 35, 33, 1285, 502, 1165, 518, 1631, 1617, 301, 145, - /* 70 */ 1159, 349, 274, 1774, 398, 36, 34, 32, 31, 30, - /* 80 */ 1, 1627, 1634, 1633, 1675, 112, 24, 132, 1647, 505, - /* 90 */ 1649, 1650, 501, 522, 522, 1157, 36, 34, 32, 31, - /* 100 */ 30, 61, 601, 1247, 271, 483, 14, 519, 35, 33, - /* 110 */ 597, 596, 1165, 1158, 108, 518, 301, 305, 1159, 104, - /* 120 */ 1181, 274, 1473, 110, 939, 128, 419, 130, 2, 1360, - /* 130 */ 54, 484, 1791, 1480, 1777, 1478, 519, 69, 212, 1722, - /* 140 */ 478, 1348, 477, 1157, 1196, 1777, 1777, 146, 104, 417, - /* 150 */ 601, 1774, 1247, 1248, 14, 424, 1160, 1471, 147, 1775, - /* 160 */ 1165, 1158, 1774, 1774, 1478, 103, 102, 101, 100, 99, - /* 170 */ 98, 97, 96, 95, 1253, 38, 2, 554, 1163, 1164, - /* 180 */ 54, 1209, 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, - /* 190 */ 1224, 1225, 1226, 1227, 1228, 1229, 553, 552, 601, 551, - /* 200 */ 550, 549, 1248, 495, 1160, 342, 1309, 341, 148, 1158, - /* 210 */ 27, 299, 1242, 1243, 1244, 1245, 1246, 1250, 1251, 1252, - /* 220 */ 131, 1183, 506, 1253, 1435, 304, 1163, 1164, 1568, 1209, + /* 0 */ 1467, 1777, 1777, 1646, 383, 1634, 384, 1380, 292, 11, + /* 10 */ 10, 343, 35, 33, 1776, 146, 24, 923, 1774, 1774, + /* 20 */ 301, 391, 1159, 384, 1380, 1631, 36, 34, 32, 31, + /* 30 */ 30, 1662, 26, 36, 34, 32, 31, 30, 518, 503, + /* 40 */ 1627, 1633, 36, 34, 32, 31, 30, 1157, 1346, 502, + /* 50 */ 1777, 522, 130, 1617, 1360, 927, 928, 518, 14, 483, + /* 60 */ 35, 33, 1285, 145, 1165, 28, 223, 1774, 301, 1675, + /* 70 */ 1159, 349, 80, 1647, 505, 1649, 1650, 501, 77, 522, + /* 80 */ 1, 62, 1715, 1777, 1181, 519, 273, 1711, 518, 1261, + /* 90 */ 1634, 113, 398, 309, 108, 1157, 1775, 104, 1777, 1470, + /* 100 */ 1774, 1646, 601, 1473, 419, 271, 14, 317, 35, 33, + /* 110 */ 1631, 147, 1165, 1158, 1478, 1774, 301, 38, 1159, 36, + /* 120 */ 34, 32, 31, 30, 388, 1627, 1633, 56, 2, 1662, + /* 130 */ 1181, 36, 34, 32, 31, 30, 522, 503, 36, 34, + /* 140 */ 32, 31, 30, 1157, 55, 1523, 1777, 502, 39, 131, + /* 150 */ 601, 1617, 291, 1435, 14, 1371, 1160, 1521, 1662, 145, + /* 160 */ 1165, 1158, 559, 1774, 1450, 274, 472, 1675, 140, 1341, + /* 170 */ 132, 1647, 505, 1649, 1650, 501, 2, 522, 1163, 1164, + /* 180 */ 1517, 1209, 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, + /* 190 */ 1224, 1225, 1226, 1227, 1228, 1229, 1247, 1410, 601, 519, + /* 200 */ 1299, 471, 1183, 55, 1160, 1617, 1456, 1196, 148, 1158, + /* 210 */ 473, 347, 447, 94, 484, 1791, 93, 92, 91, 90, + /* 220 */ 89, 88, 87, 86, 85, 1729, 1163, 1164, 1478, 1209, /* 230 */ 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, 1224, 1225, - /* 240 */ 1226, 1227, 1228, 1229, 1407, 465, 1307, 1308, 1310, 1311, - /* 250 */ 55, 54, 1160, 36, 34, 32, 31, 30, 148, 27, - /* 260 */ 299, 1242, 1243, 1244, 1245, 1246, 1250, 1251, 1252, 63, - /* 270 */ 289, 479, 1284, 188, 1163, 1164, 1346, 1209, 1210, 1212, + /* 240 */ 1226, 1227, 1228, 1229, 1454, 148, 1248, 479, 1523, 1726, + /* 250 */ 1340, 1777, 1160, 597, 596, 306, 1309, 433, 432, 55, + /* 260 */ 1521, 1523, 431, 398, 145, 109, 428, 1253, 1774, 427, + /* 270 */ 426, 425, 148, 1522, 1163, 1164, 112, 1209, 1210, 1212, /* 280 */ 1213, 1214, 1215, 1216, 498, 520, 1224, 1225, 1226, 1227, - /* 290 */ 1228, 1229, 35, 33, 36, 34, 32, 31, 30, 112, - /* 300 */ 301, 1456, 1159, 148, 577, 576, 575, 316, 148, 574, - /* 310 */ 573, 572, 114, 567, 566, 565, 564, 563, 562, 561, - /* 320 */ 560, 121, 1299, 1239, 1523, 313, 54, 1157, 991, 1662, - /* 330 */ 306, 1646, 1559, 1561, 317, 1521, 472, 110, 140, 311, - /* 340 */ 35, 33, 1230, 473, 1165, 993, 923, 128, 301, 1517, - /* 350 */ 1159, 481, 142, 1722, 1723, 1480, 1727, 26, 398, 1662, - /* 360 */ 8, 36, 34, 32, 31, 30, 503, 36, 34, 32, - /* 370 */ 31, 30, 471, 468, 1777, 1157, 502, 1410, 314, 148, - /* 380 */ 1617, 556, 601, 462, 927, 928, 128, 145, 35, 33, - /* 390 */ 334, 1774, 1165, 1158, 1480, 518, 301, 1675, 1159, 53, - /* 400 */ 268, 1647, 505, 1649, 1650, 501, 382, 522, 9, 386, - /* 410 */ 336, 332, 1029, 545, 544, 543, 1033, 542, 1035, 1036, - /* 420 */ 541, 1038, 538, 1157, 1044, 535, 1046, 1047, 532, 529, - /* 430 */ 601, 36, 34, 32, 31, 30, 1160, 433, 432, 548, - /* 440 */ 1165, 1158, 431, 474, 469, 109, 428, 1454, 519, 427, - /* 450 */ 426, 425, 1185, 1469, 148, 39, 9, 1729, 1163, 1164, - /* 460 */ 348, 1209, 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, - /* 470 */ 1224, 1225, 1226, 1227, 1228, 1229, 1478, 1283, 601, 1292, - /* 480 */ 1235, 1725, 433, 432, 1160, 1183, 1183, 431, 148, 1158, - /* 490 */ 109, 428, 1523, 447, 427, 426, 425, 1341, 312, 1637, - /* 500 */ 1196, 1159, 1617, 1521, 556, 455, 1163, 1164, 309, 1209, + /* 290 */ 1228, 1229, 35, 33, 1349, 465, 1307, 1308, 1310, 1311, + /* 300 */ 301, 556, 1159, 27, 299, 1242, 1243, 1244, 1245, 1246, + /* 310 */ 1250, 1251, 1252, 110, 939, 94, 62, 1469, 93, 92, + /* 320 */ 91, 90, 89, 88, 87, 86, 85, 1157, 143, 1722, + /* 330 */ 1723, 148, 1727, 519, 1184, 519, 479, 1631, 1474, 417, + /* 340 */ 35, 33, 1230, 506, 1165, 348, 304, 104, 301, 1568, + /* 350 */ 1159, 55, 1627, 1633, 424, 36, 34, 32, 31, 30, + /* 360 */ 8, 479, 1478, 522, 1478, 112, 36, 34, 32, 31, + /* 370 */ 30, 1556, 433, 432, 1635, 1157, 154, 431, 156, 1646, + /* 380 */ 109, 428, 601, 519, 427, 426, 425, 148, 35, 33, + /* 390 */ 112, 556, 1165, 1158, 1631, 358, 301, 305, 1159, 1523, + /* 400 */ 60, 274, 110, 59, 1182, 128, 312, 1662, 9, 1627, + /* 410 */ 1633, 1521, 1478, 1185, 1480, 503, 481, 142, 1722, 1723, + /* 420 */ 522, 1727, 468, 1157, 1196, 502, 342, 110, 341, 1617, + /* 430 */ 601, 313, 1247, 64, 289, 1397, 1160, 188, 1559, 1561, + /* 440 */ 1165, 1158, 144, 1722, 1723, 1675, 1727, 548, 263, 1647, + /* 450 */ 505, 1649, 1650, 501, 1370, 522, 9, 434, 1163, 1164, + /* 460 */ 334, 1209, 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, + /* 470 */ 1224, 1225, 1226, 1227, 1228, 1229, 283, 311, 601, 148, + /* 480 */ 336, 332, 1248, 1120, 1160, 128, 475, 455, 148, 1158, + /* 490 */ 373, 1122, 474, 469, 1480, 1463, 519, 36, 34, 32, + /* 500 */ 31, 30, 1369, 1253, 1617, 1465, 1163, 1164, 359, 1209, /* 510 */ 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, 1224, 1225, - /* 520 */ 1226, 1227, 1228, 1229, 154, 1631, 1157, 36, 34, 32, - /* 530 */ 31, 30, 1160, 1777, 390, 1639, 1523, 386, 388, 1371, - /* 540 */ 1627, 1634, 1633, 1165, 1181, 373, 145, 1522, 59, 1370, - /* 550 */ 1774, 58, 522, 456, 1163, 1164, 1646, 1209, 1210, 1212, + /* 520 */ 1226, 1227, 1228, 1229, 284, 1478, 282, 281, 1368, 421, + /* 530 */ 558, 1292, 1160, 423, 158, 157, 456, 1183, 214, 27, + /* 540 */ 299, 1242, 1243, 1244, 1245, 1246, 1250, 1251, 1252, 1646, + /* 550 */ 382, 1121, 1617, 386, 1163, 1164, 422, 1209, 1210, 1212, /* 560 */ 1213, 1214, 1215, 1216, 498, 520, 1224, 1225, 1226, 1227, - /* 570 */ 1228, 1229, 35, 33, 270, 519, 1181, 214, 1340, 61, - /* 580 */ 301, 601, 1159, 366, 1662, 519, 378, 358, 1617, 158, - /* 590 */ 157, 503, 1158, 1777, 128, 1369, 174, 516, 1617, 1729, - /* 600 */ 1474, 502, 1481, 1478, 379, 1617, 145, 1157, 139, 519, - /* 610 */ 1774, 483, 1368, 1478, 415, 411, 407, 403, 173, 283, - /* 620 */ 485, 359, 1675, 1724, 1165, 259, 1647, 505, 1649, 1650, - /* 630 */ 501, 519, 522, 1120, 940, 1160, 939, 1478, 479, 445, - /* 640 */ 2, 1122, 62, 397, 1617, 171, 1367, 1211, 1211, 11, - /* 650 */ 10, 1777, 443, 1366, 430, 429, 7, 1163, 1164, 1478, - /* 660 */ 127, 1617, 601, 941, 147, 1365, 112, 284, 1774, 282, - /* 670 */ 281, 1184, 421, 1158, 377, 1364, 423, 372, 371, 370, + /* 570 */ 1228, 1229, 35, 33, 270, 1777, 1181, 1662, 1617, 519, + /* 580 */ 301, 314, 1159, 366, 1249, 482, 378, 1461, 145, 128, + /* 590 */ 554, 397, 1774, 390, 423, 502, 386, 1235, 1480, 1617, + /* 600 */ 940, 54, 939, 1183, 379, 1254, 70, 1157, 1478, 553, + /* 610 */ 552, 1367, 551, 550, 549, 1675, 1366, 422, 81, 1647, + /* 620 */ 505, 1649, 1650, 501, 1165, 522, 1365, 1471, 1715, 941, + /* 630 */ 127, 1186, 294, 1711, 141, 1407, 32, 31, 30, 191, + /* 640 */ 2, 25, 1029, 545, 544, 543, 1033, 542, 1035, 1036, + /* 650 */ 541, 1038, 538, 1743, 1044, 535, 1046, 1047, 532, 529, + /* 660 */ 497, 1617, 601, 1364, 1363, 1362, 1617, 1359, 1358, 1357, + /* 670 */ 1356, 1355, 1354, 1158, 377, 1353, 1617, 372, 371, 370, /* 680 */ 369, 368, 365, 364, 363, 362, 361, 357, 356, 355, - /* 690 */ 354, 353, 352, 351, 350, 1617, 1183, 519, 519, 422, - /* 700 */ 519, 1121, 1617, 170, 110, 165, 1455, 167, 479, 1475, - /* 710 */ 1597, 1186, 517, 1249, 1617, 1363, 1160, 1362, 1359, 143, - /* 720 */ 1722, 1723, 487, 1727, 1617, 1478, 1478, 163, 1478, 1261, - /* 730 */ 1646, 1358, 1357, 1356, 1254, 490, 112, 1355, 1163, 1164, - /* 740 */ 1182, 1209, 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, - /* 750 */ 1224, 1225, 1226, 1227, 1228, 1229, 519, 246, 1662, 519, - /* 760 */ 1508, 1354, 1353, 1352, 1617, 503, 1617, 1617, 236, 1556, - /* 770 */ 25, 315, 571, 569, 110, 502, 156, 927, 928, 1617, - /* 780 */ 1617, 1617, 1617, 1351, 1478, 483, 1617, 1478, 506, 144, - /* 790 */ 1722, 1723, 1606, 1727, 1569, 559, 1675, 1450, 554, 79, - /* 800 */ 1647, 505, 1649, 1650, 501, 1646, 522, 423, 570, 1715, - /* 810 */ 1617, 1617, 1617, 273, 1711, 1560, 1561, 553, 552, 1397, - /* 820 */ 551, 550, 549, 1734, 1280, 1777, 1280, 558, 179, 1392, - /* 830 */ 422, 177, 1617, 1662, 1144, 1145, 199, 324, 147, 1465, - /* 840 */ 503, 434, 1774, 181, 129, 1390, 180, 1461, 337, 252, - /* 850 */ 502, 436, 183, 185, 1617, 182, 184, 118, 1211, 45, - /* 860 */ 483, 250, 52, 46, 272, 51, 1646, 439, 202, 11, - /* 870 */ 10, 1675, 1343, 1344, 79, 1647, 505, 1649, 1650, 501, - /* 880 */ 438, 522, 159, 37, 1715, 37, 191, 37, 273, 1711, - /* 890 */ 225, 497, 75, 116, 1662, 446, 547, 457, 454, 1306, - /* 900 */ 1777, 482, 71, 1361, 117, 1168, 54, 488, 204, 187, - /* 910 */ 1436, 502, 118, 145, 1167, 1617, 1453, 1774, 218, 1646, - /* 920 */ 45, 441, 491, 1255, 209, 1217, 435, 1115, 466, 1663, - /* 930 */ 227, 186, 1675, 511, 448, 80, 1647, 505, 1649, 1650, - /* 940 */ 501, 527, 522, 78, 233, 1715, 117, 1662, 1381, 294, - /* 950 */ 1711, 141, 1022, 416, 503, 50, 1518, 480, 49, 1745, - /* 960 */ 245, 217, 220, 215, 502, 118, 3, 1181, 1617, 461, - /* 970 */ 1742, 1171, 1646, 119, 57, 56, 346, 222, 117, 153, - /* 980 */ 1170, 1050, 319, 323, 340, 1675, 1054, 964, 263, 1647, - /* 990 */ 505, 1649, 1650, 501, 279, 522, 269, 991, 280, 330, - /* 1000 */ 1662, 326, 322, 150, 965, 1061, 241, 482, 554, 1128, - /* 1010 */ 360, 155, 1558, 1059, 367, 374, 375, 502, 120, 376, - /* 1020 */ 380, 1617, 1187, 381, 389, 1190, 475, 553, 552, 392, - /* 1030 */ 551, 550, 549, 393, 148, 162, 164, 1189, 1675, 1349, - /* 1040 */ 394, 80, 1647, 505, 1649, 1650, 501, 1646, 522, 166, - /* 1050 */ 395, 1715, 1188, 396, 169, 294, 1711, 141, 60, 399, - /* 1060 */ 94, 172, 420, 93, 92, 91, 90, 89, 88, 87, - /* 1070 */ 86, 85, 1468, 418, 288, 1662, 1743, 1165, 176, 1464, - /* 1080 */ 1646, 84, 503, 178, 122, 1601, 123, 242, 189, 1466, - /* 1090 */ 1462, 124, 502, 125, 449, 1186, 1617, 450, 192, 453, - /* 1100 */ 1746, 243, 194, 467, 458, 197, 1756, 509, 1662, 6, - /* 1110 */ 459, 1755, 476, 1675, 1736, 503, 80, 1647, 505, 1649, - /* 1120 */ 1650, 501, 464, 522, 293, 502, 1715, 200, 470, 1617, - /* 1130 */ 294, 1711, 1790, 463, 203, 5, 208, 1280, 135, 210, - /* 1140 */ 111, 1749, 1646, 1185, 40, 492, 1675, 295, 1730, 80, - /* 1150 */ 1647, 505, 1649, 1650, 501, 216, 522, 1567, 489, 1715, - /* 1160 */ 18, 1566, 211, 294, 1711, 1790, 512, 229, 507, 244, - /* 1170 */ 1662, 508, 303, 513, 1772, 514, 70, 503, 231, 1696, - /* 1180 */ 68, 525, 1793, 1479, 247, 1451, 238, 502, 600, 1611, - /* 1190 */ 290, 1617, 47, 1773, 249, 134, 1646, 253, 251, 1610, - /* 1200 */ 486, 219, 260, 318, 221, 1646, 254, 1607, 1675, 493, - /* 1210 */ 320, 80, 1647, 505, 1649, 1650, 501, 321, 522, 1153, - /* 1220 */ 1154, 1715, 151, 325, 1662, 294, 1711, 1790, 1605, 327, - /* 1230 */ 328, 503, 1604, 1662, 329, 331, 1733, 1603, 333, 1602, - /* 1240 */ 503, 502, 335, 1587, 152, 1617, 338, 339, 1131, 1130, - /* 1250 */ 502, 1581, 1580, 344, 1617, 1579, 345, 1578, 1098, 1646, - /* 1260 */ 483, 1551, 1675, 1550, 1549, 132, 1647, 505, 1649, 1650, - /* 1270 */ 501, 1675, 522, 1548, 259, 1647, 505, 1649, 1650, 501, - /* 1280 */ 1547, 522, 1546, 1545, 1544, 1543, 1542, 1662, 1541, 1540, - /* 1290 */ 1539, 1538, 1537, 1536, 503, 1535, 1534, 1533, 115, 1646, - /* 1300 */ 1777, 1532, 1531, 1530, 502, 1529, 1528, 1527, 1617, 1100, - /* 1310 */ 1792, 1526, 1525, 145, 1524, 1409, 1377, 1774, 1646, 160, - /* 1320 */ 1376, 385, 138, 401, 930, 1675, 1595, 1662, 81, 1647, - /* 1330 */ 505, 1649, 1650, 501, 503, 522, 106, 929, 1715, 405, - /* 1340 */ 1402, 107, 1714, 1711, 502, 1589, 1662, 1573, 1617, 1564, - /* 1350 */ 1457, 161, 387, 500, 168, 1408, 1406, 958, 402, 1404, - /* 1360 */ 1400, 406, 410, 502, 414, 1675, 400, 1617, 81, 1647, - /* 1370 */ 505, 1649, 1650, 501, 1646, 522, 404, 408, 1715, 409, - /* 1380 */ 412, 413, 494, 1711, 1675, 1389, 1388, 267, 1647, 505, - /* 1390 */ 1649, 1650, 501, 499, 522, 496, 1687, 1375, 1459, 1065, - /* 1400 */ 1064, 1458, 1662, 604, 990, 989, 568, 83, 988, 503, - /* 1410 */ 987, 570, 1398, 984, 983, 175, 285, 240, 982, 502, - /* 1420 */ 437, 286, 1391, 1617, 1393, 287, 440, 1374, 442, 105, - /* 1430 */ 1373, 444, 1594, 1646, 82, 593, 589, 585, 581, 239, - /* 1440 */ 1675, 1588, 1138, 81, 1647, 505, 1649, 1650, 501, 1572, - /* 1450 */ 522, 451, 1571, 1715, 1563, 64, 196, 4, 1712, 37, - /* 1460 */ 15, 1662, 201, 77, 43, 207, 234, 48, 503, 1305, - /* 1470 */ 133, 205, 22, 41, 1298, 206, 452, 193, 502, 1637, - /* 1480 */ 65, 198, 1617, 23, 16, 298, 1277, 1646, 1276, 213, - /* 1490 */ 136, 308, 307, 126, 42, 1329, 1334, 17, 1323, 1675, - /* 1500 */ 515, 1173, 268, 1647, 505, 1649, 1650, 501, 1328, 522, - /* 1510 */ 296, 1333, 13, 1332, 297, 1662, 10, 1646, 19, 1219, - /* 1520 */ 1240, 1218, 500, 29, 137, 460, 1166, 12, 195, 149, - /* 1530 */ 20, 1204, 502, 504, 1562, 21, 1617, 230, 224, 1303, - /* 1540 */ 226, 71, 228, 1165, 1175, 1662, 1136, 510, 190, 66, - /* 1550 */ 1646, 232, 503, 1675, 67, 1636, 267, 1647, 505, 1649, - /* 1560 */ 1650, 501, 502, 522, 235, 1688, 1617, 1678, 521, 300, - /* 1570 */ 1221, 44, 524, 1051, 526, 310, 528, 530, 1662, 1048, - /* 1580 */ 1646, 523, 531, 1675, 1045, 503, 268, 1647, 505, 1649, - /* 1590 */ 1650, 501, 1169, 522, 533, 502, 1039, 534, 536, 1617, - /* 1600 */ 1037, 537, 302, 539, 540, 1028, 1060, 1058, 1662, 1043, - /* 1610 */ 1646, 1042, 479, 72, 1041, 503, 1675, 1040, 546, 268, - /* 1620 */ 1647, 505, 1649, 1650, 501, 502, 522, 1057, 73, 1617, - /* 1630 */ 74, 1056, 956, 555, 997, 1174, 557, 237, 1662, 978, - /* 1640 */ 112, 977, 976, 973, 975, 503, 1675, 974, 972, 255, - /* 1650 */ 1647, 505, 1649, 1650, 501, 502, 522, 1177, 971, 1617, - /* 1660 */ 483, 1646, 994, 992, 968, 967, 966, 963, 520, 1224, - /* 1670 */ 1225, 962, 961, 1405, 578, 579, 1675, 1646, 110, 262, - /* 1680 */ 1647, 505, 1649, 1650, 501, 580, 522, 1403, 582, 1662, - /* 1690 */ 583, 584, 1401, 212, 1722, 478, 503, 477, 586, 588, - /* 1700 */ 1777, 1399, 587, 591, 592, 1662, 502, 590, 1387, 595, - /* 1710 */ 1617, 594, 503, 145, 1386, 1372, 598, 1774, 599, 1347, - /* 1720 */ 1161, 248, 502, 602, 1646, 603, 1617, 1675, 1347, 1347, - /* 1730 */ 264, 1647, 505, 1649, 1650, 501, 1347, 522, 1347, 1347, - /* 1740 */ 1347, 1347, 1646, 1675, 1347, 1347, 256, 1647, 505, 1649, - /* 1750 */ 1650, 501, 1662, 522, 1347, 1347, 1347, 1646, 1347, 503, - /* 1760 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 502, - /* 1770 */ 1662, 1347, 1347, 1617, 1347, 1347, 1347, 503, 1347, 1347, - /* 1780 */ 1347, 1347, 1347, 1347, 1347, 1662, 1347, 502, 1347, 1347, - /* 1790 */ 1675, 1617, 503, 265, 1647, 505, 1649, 1650, 501, 1347, - /* 1800 */ 522, 1347, 502, 1347, 1646, 1347, 1617, 1347, 1675, 1347, - /* 1810 */ 1347, 257, 1647, 505, 1649, 1650, 501, 1347, 522, 1347, - /* 1820 */ 1646, 1347, 1347, 1675, 1347, 1347, 266, 1647, 505, 1649, - /* 1830 */ 1650, 501, 1662, 522, 1347, 1347, 1347, 1347, 1347, 503, - /* 1840 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1662, 502, - /* 1850 */ 1347, 1347, 1347, 1617, 1347, 503, 1347, 1347, 1347, 1347, - /* 1860 */ 1347, 1347, 1347, 1347, 1347, 502, 1347, 1646, 1347, 1617, - /* 1870 */ 1675, 1347, 1347, 258, 1647, 505, 1649, 1650, 501, 1347, - /* 1880 */ 522, 1347, 1347, 1347, 1347, 1347, 1675, 1347, 1347, 1658, - /* 1890 */ 1647, 505, 1649, 1650, 501, 1662, 522, 1646, 1347, 1347, - /* 1900 */ 1347, 1347, 503, 1347, 1347, 1347, 1347, 1347, 1347, 1347, - /* 1910 */ 1347, 1347, 502, 1347, 1347, 1347, 1617, 1347, 1347, 1347, - /* 1920 */ 1347, 1347, 1347, 1347, 1347, 1662, 1347, 1347, 1347, 1347, - /* 1930 */ 1347, 1347, 503, 1675, 1347, 1347, 1657, 1647, 505, 1649, - /* 1940 */ 1650, 501, 502, 522, 1347, 1347, 1617, 1646, 1347, 1347, - /* 1950 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, - /* 1960 */ 1347, 1347, 1347, 1675, 1347, 1347, 1656, 1647, 505, 1649, - /* 1970 */ 1650, 501, 1347, 522, 1347, 1662, 1347, 1646, 1347, 1347, - /* 1980 */ 1347, 1347, 503, 1347, 1347, 1347, 1347, 1347, 1347, 1347, - /* 1990 */ 1347, 1347, 502, 1347, 1347, 1347, 1617, 1347, 1347, 1347, - /* 2000 */ 1347, 1347, 1347, 1347, 1347, 1662, 1347, 1347, 1347, 1347, - /* 2010 */ 1347, 1347, 503, 1675, 1347, 1347, 277, 1647, 505, 1649, - /* 2020 */ 1650, 501, 502, 522, 1347, 1347, 1617, 1646, 1347, 1347, - /* 2030 */ 1347, 1347, 1347, 1347, 1347, 1347, 1646, 1347, 1347, 1347, - /* 2040 */ 1347, 1347, 1347, 1675, 1347, 1347, 276, 1647, 505, 1649, - /* 2050 */ 1650, 501, 1347, 522, 1347, 1662, 1347, 1347, 1347, 1347, - /* 2060 */ 1347, 1347, 503, 1347, 1662, 1347, 1347, 1347, 1347, 1347, - /* 2070 */ 1347, 503, 502, 1347, 1347, 1347, 1617, 1347, 1347, 1347, - /* 2080 */ 1347, 502, 1347, 1347, 1347, 1617, 1347, 1347, 1347, 1347, - /* 2090 */ 1646, 1347, 1347, 1675, 1347, 1347, 278, 1647, 505, 1649, - /* 2100 */ 1650, 501, 1675, 522, 1347, 275, 1647, 505, 1649, 1650, - /* 2110 */ 501, 1347, 522, 1347, 1347, 1347, 1347, 1347, 1662, 1347, - /* 2120 */ 1347, 1347, 1347, 1347, 1347, 503, 1347, 1347, 1347, 1347, - /* 2130 */ 1347, 1347, 1347, 1347, 1347, 502, 1347, 1347, 1347, 1617, - /* 2140 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, - /* 2150 */ 1347, 1347, 1347, 1347, 1347, 1347, 1675, 1347, 1347, 261, - /* 2160 */ 1647, 505, 1649, 1650, 501, 1347, 522, + /* 690 */ 354, 353, 352, 351, 350, 577, 576, 575, 316, 1211, + /* 700 */ 574, 573, 572, 114, 567, 566, 565, 564, 563, 562, + /* 710 */ 561, 560, 121, 1617, 1617, 1617, 1160, 1617, 1617, 1617, + /* 720 */ 1617, 1617, 1617, 1352, 7, 1617, 128, 1351, 430, 429, + /* 730 */ 1646, 571, 569, 1560, 1561, 1481, 927, 928, 1163, 1164, + /* 740 */ 1729, 1209, 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, + /* 750 */ 1224, 1225, 1226, 1227, 1228, 1229, 199, 129, 1662, 519, + /* 760 */ 1284, 1646, 252, 1729, 1725, 1211, 503, 991, 1159, 519, + /* 770 */ 1183, 1475, 1606, 1617, 250, 53, 502, 1617, 52, 506, + /* 780 */ 1617, 1597, 1734, 1280, 993, 1569, 483, 1724, 1478, 1662, + /* 790 */ 1144, 1145, 487, 1157, 479, 159, 1675, 482, 1478, 80, + /* 800 */ 1647, 505, 1649, 1650, 501, 246, 522, 502, 1508, 1715, + /* 810 */ 1165, 1617, 519, 273, 1711, 570, 454, 324, 179, 55, + /* 820 */ 1168, 177, 485, 112, 516, 1777, 181, 1675, 490, 180, + /* 830 */ 81, 1647, 505, 1649, 1650, 501, 1637, 522, 145, 519, + /* 840 */ 1715, 1478, 1774, 483, 294, 1711, 141, 183, 601, 185, + /* 850 */ 182, 517, 184, 445, 495, 337, 79, 1646, 215, 1158, + /* 860 */ 110, 519, 519, 547, 461, 1742, 443, 964, 1478, 47, + /* 870 */ 272, 118, 1639, 236, 315, 212, 1722, 478, 1392, 477, + /* 880 */ 11, 10, 1777, 1390, 965, 1662, 1171, 58, 57, 346, + /* 890 */ 1478, 1478, 153, 503, 1280, 147, 1361, 340, 46, 1774, + /* 900 */ 436, 1167, 1160, 502, 202, 439, 37, 1617, 1436, 269, + /* 910 */ 37, 457, 330, 37, 326, 322, 150, 225, 1343, 1344, + /* 920 */ 1646, 1455, 116, 1675, 1163, 1164, 81, 1647, 505, 1649, + /* 930 */ 1650, 501, 1211, 522, 218, 117, 1715, 466, 1306, 76, + /* 940 */ 294, 1711, 1790, 448, 204, 118, 1255, 148, 1662, 72, + /* 950 */ 1217, 1749, 1663, 1115, 1381, 46, 503, 227, 527, 209, + /* 960 */ 416, 174, 511, 480, 1518, 1283, 502, 1170, 1745, 217, + /* 970 */ 1617, 117, 1646, 139, 1239, 233, 220, 488, 222, 415, + /* 980 */ 411, 407, 403, 173, 118, 1022, 1675, 119, 117, 81, + /* 990 */ 1647, 505, 1649, 1650, 501, 245, 522, 1453, 1050, 1715, + /* 1000 */ 1662, 3, 1181, 294, 1711, 1790, 319, 63, 503, 323, + /* 1010 */ 171, 1054, 991, 554, 1772, 491, 280, 279, 502, 241, + /* 1020 */ 1128, 155, 1617, 360, 1061, 1558, 367, 1059, 120, 375, + /* 1030 */ 374, 1646, 553, 552, 380, 551, 550, 549, 1675, 1187, + /* 1040 */ 376, 81, 1647, 505, 1649, 1650, 501, 438, 522, 381, + /* 1050 */ 389, 1715, 1190, 162, 392, 294, 1711, 1790, 393, 1662, + /* 1060 */ 1189, 164, 446, 394, 166, 395, 1733, 503, 170, 1188, + /* 1070 */ 165, 396, 167, 399, 169, 61, 187, 502, 418, 172, + /* 1080 */ 1165, 1617, 1646, 420, 1468, 176, 1464, 483, 441, 554, + /* 1090 */ 84, 242, 163, 435, 288, 1601, 178, 1675, 186, 1646, + /* 1100 */ 259, 1647, 505, 1649, 1650, 501, 122, 522, 553, 552, + /* 1110 */ 1662, 551, 550, 549, 123, 1466, 1462, 124, 503, 125, + /* 1120 */ 449, 189, 51, 453, 450, 50, 1777, 1662, 502, 243, + /* 1130 */ 458, 192, 1617, 194, 1186, 503, 197, 459, 483, 147, + /* 1140 */ 1746, 467, 509, 1774, 1756, 502, 6, 200, 1675, 1617, + /* 1150 */ 463, 259, 1647, 505, 1649, 1650, 501, 1646, 522, 464, + /* 1160 */ 476, 5, 1736, 203, 1755, 1675, 293, 210, 82, 1647, + /* 1170 */ 505, 1649, 1650, 501, 1280, 522, 111, 1777, 1715, 470, + /* 1180 */ 208, 1185, 1714, 1711, 1348, 1662, 40, 211, 492, 1646, + /* 1190 */ 145, 1773, 135, 503, 1774, 1730, 295, 489, 18, 1567, + /* 1200 */ 1793, 507, 508, 502, 1566, 512, 303, 1617, 103, 102, + /* 1210 */ 101, 100, 99, 98, 97, 96, 95, 1662, 216, 479, + /* 1220 */ 1696, 513, 229, 1675, 219, 500, 82, 1647, 505, 1649, + /* 1230 */ 1650, 501, 514, 522, 231, 502, 1715, 69, 486, 1617, + /* 1240 */ 494, 1711, 1646, 493, 221, 244, 1479, 71, 112, 525, + /* 1250 */ 1451, 247, 600, 238, 48, 1675, 134, 253, 267, 1647, + /* 1260 */ 505, 1649, 1650, 501, 499, 522, 496, 1687, 483, 290, + /* 1270 */ 1662, 260, 249, 254, 251, 1611, 1610, 318, 503, 1607, + /* 1280 */ 320, 321, 1153, 1154, 151, 110, 325, 1605, 502, 327, + /* 1290 */ 328, 329, 1617, 1604, 331, 1603, 333, 1602, 335, 1646, + /* 1300 */ 212, 1722, 478, 1587, 477, 152, 339, 1777, 1675, 338, + /* 1310 */ 1131, 82, 1647, 505, 1649, 1650, 501, 1581, 522, 1130, + /* 1320 */ 145, 1715, 1580, 344, 1774, 345, 1712, 1662, 604, 1579, + /* 1330 */ 1578, 1551, 1098, 1550, 1549, 503, 1548, 1547, 1546, 1545, + /* 1340 */ 1544, 1543, 240, 1100, 115, 502, 1646, 1542, 1541, 1617, + /* 1350 */ 1540, 1539, 462, 1538, 105, 1537, 1536, 1535, 1534, 1533, + /* 1360 */ 593, 589, 585, 581, 239, 1675, 1532, 1531, 268, 1647, + /* 1370 */ 505, 1649, 1650, 501, 1662, 522, 1530, 1529, 1528, 1527, + /* 1380 */ 1526, 1525, 503, 1524, 1409, 1646, 1377, 138, 78, 160, + /* 1390 */ 1376, 234, 502, 1595, 1589, 1573, 1617, 106, 385, 930, + /* 1400 */ 161, 929, 107, 1564, 1457, 387, 168, 1408, 1406, 401, + /* 1410 */ 400, 958, 1675, 1662, 1404, 132, 1647, 505, 1649, 1650, + /* 1420 */ 501, 503, 522, 1402, 1400, 515, 402, 1389, 406, 404, + /* 1430 */ 1388, 502, 175, 405, 410, 1617, 1646, 414, 298, 408, + /* 1440 */ 1375, 409, 1459, 1458, 413, 412, 1398, 1064, 1065, 990, + /* 1450 */ 460, 1675, 989, 195, 268, 1647, 505, 1649, 1650, 501, + /* 1460 */ 1792, 522, 45, 988, 1662, 1646, 987, 568, 1393, 570, + /* 1470 */ 285, 1136, 500, 190, 286, 1391, 984, 287, 983, 1374, + /* 1480 */ 982, 440, 502, 437, 442, 1373, 1617, 444, 1594, 83, + /* 1490 */ 1588, 1138, 451, 1662, 1572, 1571, 126, 1646, 1563, 4, + /* 1500 */ 65, 503, 1675, 196, 37, 267, 1647, 505, 1649, 1650, + /* 1510 */ 501, 502, 522, 49, 1688, 1617, 452, 193, 300, 15, + /* 1520 */ 201, 43, 1305, 206, 41, 1662, 1298, 133, 207, 205, + /* 1530 */ 22, 1675, 23, 503, 268, 1647, 505, 1649, 1650, 501, + /* 1540 */ 1637, 522, 1277, 502, 66, 213, 198, 1617, 1276, 42, + /* 1550 */ 302, 136, 1334, 1646, 16, 17, 13, 1323, 1329, 10, + /* 1560 */ 1328, 19, 296, 1675, 1333, 1332, 268, 1647, 505, 1649, + /* 1570 */ 1650, 501, 297, 522, 1219, 137, 149, 29, 1204, 510, + /* 1580 */ 1218, 1662, 12, 20, 1646, 21, 226, 1240, 1562, 503, + /* 1590 */ 504, 224, 230, 232, 72, 1636, 235, 1303, 1175, 502, + /* 1600 */ 1221, 228, 67, 1617, 68, 526, 1678, 521, 44, 310, + /* 1610 */ 1051, 1048, 1662, 1646, 524, 528, 530, 531, 533, 1675, + /* 1620 */ 503, 1045, 255, 1647, 505, 1649, 1650, 501, 534, 522, + /* 1630 */ 502, 1039, 536, 537, 1617, 1037, 539, 1043, 1042, 540, + /* 1640 */ 1041, 1662, 1040, 1028, 73, 74, 1060, 75, 1057, 503, + /* 1650 */ 1675, 1056, 546, 262, 1647, 505, 1649, 1650, 501, 502, + /* 1660 */ 522, 956, 1646, 1617, 555, 557, 237, 997, 308, 307, + /* 1670 */ 978, 977, 973, 1646, 1058, 976, 994, 975, 1173, 1675, + /* 1680 */ 974, 972, 264, 1647, 505, 1649, 1650, 501, 971, 522, + /* 1690 */ 1662, 992, 968, 967, 966, 963, 962, 1405, 503, 961, + /* 1700 */ 578, 1662, 579, 1166, 580, 1403, 582, 583, 502, 503, + /* 1710 */ 584, 1401, 1617, 586, 587, 588, 1399, 590, 592, 502, + /* 1720 */ 1165, 591, 1387, 1617, 1646, 1386, 594, 1372, 1675, 595, + /* 1730 */ 598, 256, 1647, 505, 1649, 1650, 501, 1161, 522, 1675, + /* 1740 */ 599, 603, 265, 1647, 505, 1649, 1650, 501, 248, 522, + /* 1750 */ 602, 1347, 1662, 1646, 1347, 1347, 1347, 1347, 523, 1347, + /* 1760 */ 503, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1169, + /* 1770 */ 502, 1347, 1347, 1347, 1617, 1347, 1347, 1347, 1347, 1347, + /* 1780 */ 1347, 1662, 1347, 1347, 1347, 1646, 1347, 1347, 1347, 503, + /* 1790 */ 1675, 1347, 1347, 257, 1647, 505, 1649, 1650, 501, 502, + /* 1800 */ 522, 1347, 1347, 1617, 1347, 1347, 1347, 1347, 1347, 1347, + /* 1810 */ 1347, 1347, 1174, 1662, 1347, 1347, 1347, 1347, 1347, 1675, + /* 1820 */ 1347, 503, 266, 1647, 505, 1649, 1650, 501, 1347, 522, + /* 1830 */ 1347, 502, 1347, 1347, 1177, 1617, 1347, 1347, 1347, 1347, + /* 1840 */ 1347, 1646, 1347, 1347, 1347, 520, 1224, 1225, 1347, 1347, + /* 1850 */ 1347, 1675, 1347, 1347, 258, 1647, 505, 1649, 1650, 501, + /* 1860 */ 1347, 522, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1662, + /* 1870 */ 1347, 1347, 1646, 1347, 1347, 1347, 1347, 503, 1347, 1347, + /* 1880 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 502, 1347, 1347, + /* 1890 */ 1347, 1617, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, + /* 1900 */ 1662, 1646, 1347, 1347, 1347, 1347, 1347, 1675, 503, 1347, + /* 1910 */ 1658, 1647, 505, 1649, 1650, 501, 1347, 522, 502, 1347, + /* 1920 */ 1347, 1347, 1617, 1347, 1347, 1347, 1347, 1347, 1347, 1662, + /* 1930 */ 1347, 1347, 1646, 1347, 1347, 1347, 1347, 503, 1675, 1347, + /* 1940 */ 1347, 1657, 1647, 505, 1649, 1650, 501, 502, 522, 1347, + /* 1950 */ 1347, 1617, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, + /* 1960 */ 1662, 1646, 1347, 1347, 1347, 1347, 1347, 1675, 503, 1347, + /* 1970 */ 1656, 1647, 505, 1649, 1650, 501, 1347, 522, 502, 1347, + /* 1980 */ 1347, 1347, 1617, 1347, 1347, 1347, 1347, 1347, 1347, 1662, + /* 1990 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 503, 1675, 1347, + /* 2000 */ 1347, 277, 1647, 505, 1649, 1650, 501, 502, 522, 1347, + /* 2010 */ 1347, 1617, 1646, 1347, 1347, 1347, 1347, 1347, 1347, 1347, + /* 2020 */ 1347, 1347, 1347, 1646, 1347, 1347, 1347, 1675, 1347, 1347, + /* 2030 */ 276, 1647, 505, 1649, 1650, 501, 1347, 522, 1347, 1347, + /* 2040 */ 1662, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 503, 1347, + /* 2050 */ 1347, 1662, 1347, 1347, 1347, 1347, 1347, 1347, 502, 503, + /* 2060 */ 1347, 1347, 1617, 1347, 1347, 1347, 1347, 1347, 1347, 502, + /* 2070 */ 1347, 1347, 1347, 1617, 1347, 1347, 1347, 1646, 1675, 1347, + /* 2080 */ 1347, 278, 1647, 505, 1649, 1650, 501, 1347, 522, 1675, + /* 2090 */ 1347, 1347, 275, 1647, 505, 1649, 1650, 501, 1347, 522, + /* 2100 */ 1347, 1347, 1347, 1347, 1347, 1662, 1347, 1347, 1347, 1347, + /* 2110 */ 1347, 1347, 1347, 503, 1347, 1347, 1347, 1347, 1347, 1347, + /* 2120 */ 1347, 1347, 1347, 502, 1347, 1347, 1347, 1617, 1347, 1347, + /* 2130 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, + /* 2140 */ 1347, 1347, 1347, 1675, 1347, 1347, 261, 1647, 505, 1649, + /* 2150 */ 1650, 501, 1347, 522, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 244, 251, 246, 247, 244, 248, 246, 247, 290, 321, - /* 10 */ 322, 270, 12, 13, 264, 270, 296, 260, 241, 336, - /* 20 */ 20, 271, 22, 305, 306, 307, 12, 13, 14, 15, - /* 30 */ 16, 308, 349, 276, 21, 317, 353, 24, 25, 26, - /* 40 */ 27, 28, 29, 30, 31, 32, 269, 47, 269, 273, - /* 50 */ 14, 15, 16, 276, 275, 332, 336, 248, 58, 280, - /* 60 */ 12, 13, 14, 286, 64, 20, 290, 290, 20, 349, - /* 70 */ 22, 248, 58, 353, 57, 12, 13, 14, 15, 16, - /* 80 */ 80, 305, 306, 307, 307, 276, 2, 310, 311, 312, - /* 90 */ 313, 314, 315, 317, 317, 47, 12, 13, 14, 15, - /* 100 */ 16, 253, 102, 89, 281, 296, 58, 248, 12, 13, - /* 110 */ 249, 250, 64, 113, 266, 20, 20, 261, 22, 260, - /* 120 */ 20, 58, 274, 314, 22, 269, 267, 240, 80, 242, - /* 130 */ 80, 354, 355, 277, 336, 276, 248, 251, 329, 330, - /* 140 */ 331, 0, 333, 47, 81, 336, 336, 349, 260, 47, - /* 150 */ 102, 353, 89, 139, 58, 267, 156, 271, 349, 349, - /* 160 */ 64, 113, 353, 353, 276, 24, 25, 26, 27, 28, - /* 170 */ 29, 30, 31, 32, 160, 80, 80, 92, 178, 179, - /* 180 */ 80, 181, 182, 183, 184, 185, 186, 187, 188, 189, - /* 190 */ 190, 191, 192, 193, 194, 195, 111, 112, 102, 114, - /* 200 */ 115, 116, 139, 58, 156, 155, 178, 157, 208, 113, - /* 210 */ 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, - /* 220 */ 254, 20, 286, 160, 258, 289, 178, 179, 292, 181, + /* 0 */ 270, 336, 336, 241, 244, 271, 246, 247, 274, 1, + /* 10 */ 2, 297, 12, 13, 349, 349, 2, 4, 353, 353, + /* 20 */ 20, 244, 22, 246, 247, 291, 12, 13, 14, 15, + /* 30 */ 16, 269, 2, 12, 13, 14, 15, 16, 20, 277, + /* 40 */ 306, 307, 12, 13, 14, 15, 16, 47, 238, 287, + /* 50 */ 336, 317, 240, 291, 242, 42, 43, 20, 58, 297, + /* 60 */ 12, 13, 14, 349, 64, 321, 322, 353, 20, 307, + /* 70 */ 22, 248, 310, 311, 312, 313, 314, 315, 251, 317, + /* 80 */ 80, 253, 320, 336, 20, 248, 324, 325, 20, 81, + /* 90 */ 271, 264, 57, 274, 266, 47, 349, 260, 336, 272, + /* 100 */ 353, 241, 102, 275, 267, 282, 58, 297, 12, 13, + /* 110 */ 291, 349, 64, 113, 277, 353, 20, 80, 22, 12, + /* 120 */ 13, 14, 15, 16, 14, 306, 307, 4, 80, 269, + /* 130 */ 20, 12, 13, 14, 15, 16, 317, 277, 12, 13, + /* 140 */ 14, 15, 16, 47, 80, 269, 336, 287, 80, 254, + /* 150 */ 102, 291, 276, 258, 58, 241, 156, 281, 269, 349, + /* 160 */ 64, 113, 257, 353, 259, 58, 277, 307, 268, 148, + /* 170 */ 310, 311, 312, 313, 314, 315, 80, 317, 178, 179, + /* 180 */ 280, 181, 182, 183, 184, 185, 186, 187, 188, 189, + /* 190 */ 190, 191, 192, 193, 194, 195, 89, 0, 102, 248, + /* 200 */ 81, 312, 20, 80, 156, 291, 0, 81, 208, 113, + /* 210 */ 20, 260, 297, 21, 354, 355, 24, 25, 26, 27, + /* 220 */ 28, 29, 30, 31, 32, 308, 178, 179, 277, 181, /* 230 */ 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - /* 240 */ 192, 193, 194, 195, 0, 217, 218, 219, 220, 221, - /* 250 */ 4, 80, 156, 12, 13, 14, 15, 16, 208, 196, - /* 260 */ 197, 198, 199, 200, 201, 202, 203, 204, 205, 165, - /* 270 */ 166, 248, 4, 169, 178, 179, 238, 181, 182, 183, + /* 240 */ 192, 193, 194, 195, 0, 208, 139, 248, 269, 332, + /* 250 */ 229, 336, 156, 249, 250, 276, 178, 60, 61, 80, + /* 260 */ 281, 269, 65, 57, 349, 68, 69, 160, 353, 72, + /* 270 */ 73, 74, 208, 281, 178, 179, 277, 181, 182, 183, /* 280 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - /* 290 */ 194, 195, 12, 13, 12, 13, 14, 15, 16, 276, - /* 300 */ 20, 0, 22, 208, 60, 61, 62, 63, 208, 65, - /* 310 */ 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, - /* 320 */ 76, 77, 81, 178, 269, 278, 80, 47, 47, 269, - /* 330 */ 275, 241, 285, 286, 296, 280, 276, 314, 268, 261, - /* 340 */ 12, 13, 14, 20, 64, 64, 4, 269, 20, 279, - /* 350 */ 22, 328, 329, 330, 331, 277, 333, 2, 57, 269, - /* 360 */ 80, 12, 13, 14, 15, 16, 276, 12, 13, 14, - /* 370 */ 15, 16, 312, 143, 336, 47, 286, 0, 261, 208, - /* 380 */ 290, 57, 102, 293, 42, 43, 269, 349, 12, 13, - /* 390 */ 151, 353, 64, 113, 277, 20, 20, 307, 22, 3, - /* 400 */ 310, 311, 312, 313, 314, 315, 245, 317, 80, 248, - /* 410 */ 171, 172, 93, 94, 95, 96, 97, 98, 99, 100, - /* 420 */ 101, 102, 103, 47, 105, 106, 107, 108, 109, 110, - /* 430 */ 102, 12, 13, 14, 15, 16, 156, 60, 61, 91, - /* 440 */ 64, 113, 65, 213, 214, 68, 69, 0, 248, 72, - /* 450 */ 73, 74, 20, 241, 208, 80, 80, 308, 178, 179, - /* 460 */ 260, 181, 182, 183, 184, 185, 186, 187, 188, 189, - /* 470 */ 190, 191, 192, 193, 194, 195, 276, 209, 102, 14, - /* 480 */ 14, 332, 60, 61, 156, 20, 20, 65, 208, 113, - /* 490 */ 68, 69, 269, 296, 72, 73, 74, 148, 275, 44, - /* 500 */ 81, 22, 290, 280, 57, 248, 178, 179, 273, 181, + /* 290 */ 194, 195, 12, 13, 0, 217, 218, 219, 220, 221, + /* 300 */ 20, 57, 22, 196, 197, 198, 199, 200, 201, 202, + /* 310 */ 203, 204, 205, 314, 22, 21, 253, 271, 24, 25, + /* 320 */ 26, 27, 28, 29, 30, 31, 32, 47, 329, 330, + /* 330 */ 331, 208, 333, 248, 20, 248, 248, 291, 275, 47, + /* 340 */ 12, 13, 14, 287, 64, 260, 290, 260, 20, 293, + /* 350 */ 22, 80, 306, 307, 267, 12, 13, 14, 15, 16, + /* 360 */ 80, 248, 277, 317, 277, 277, 12, 13, 14, 15, + /* 370 */ 16, 277, 60, 61, 271, 47, 55, 65, 284, 241, + /* 380 */ 68, 69, 102, 248, 72, 73, 74, 208, 12, 13, + /* 390 */ 277, 57, 64, 113, 291, 260, 20, 261, 22, 269, + /* 400 */ 79, 58, 314, 82, 20, 269, 276, 269, 80, 306, + /* 410 */ 307, 281, 277, 20, 278, 277, 328, 329, 330, 331, + /* 420 */ 317, 333, 143, 47, 81, 287, 155, 314, 157, 291, + /* 430 */ 102, 279, 89, 165, 166, 0, 156, 169, 286, 287, + /* 440 */ 64, 113, 329, 330, 331, 307, 333, 91, 310, 311, + /* 450 */ 312, 313, 314, 315, 241, 317, 80, 22, 178, 179, + /* 460 */ 151, 181, 182, 183, 184, 185, 186, 187, 188, 189, + /* 470 */ 190, 191, 192, 193, 194, 195, 35, 261, 102, 208, + /* 480 */ 171, 172, 139, 79, 156, 269, 348, 248, 208, 113, + /* 490 */ 75, 87, 213, 214, 278, 270, 248, 12, 13, 14, + /* 500 */ 15, 16, 241, 160, 291, 270, 178, 179, 260, 181, /* 510 */ 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - /* 520 */ 192, 193, 194, 195, 55, 290, 47, 12, 13, 14, - /* 530 */ 15, 16, 156, 336, 245, 80, 269, 248, 14, 241, - /* 540 */ 305, 306, 307, 64, 20, 75, 349, 280, 79, 241, - /* 550 */ 353, 82, 317, 296, 178, 179, 241, 181, 182, 183, + /* 520 */ 192, 193, 194, 195, 83, 277, 85, 86, 241, 88, + /* 530 */ 64, 14, 156, 92, 119, 120, 297, 20, 145, 196, + /* 540 */ 197, 198, 199, 200, 201, 202, 203, 204, 205, 241, + /* 550 */ 245, 147, 291, 248, 178, 179, 115, 181, 182, 183, /* 560 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - /* 570 */ 194, 195, 12, 13, 18, 248, 20, 145, 229, 253, - /* 580 */ 20, 102, 22, 27, 269, 248, 30, 260, 290, 119, - /* 590 */ 120, 276, 113, 336, 269, 241, 33, 260, 290, 308, - /* 600 */ 274, 286, 277, 276, 48, 290, 349, 47, 45, 248, - /* 610 */ 353, 296, 241, 276, 51, 52, 53, 54, 55, 35, - /* 620 */ 224, 260, 307, 332, 64, 310, 311, 312, 313, 314, - /* 630 */ 315, 248, 317, 79, 20, 156, 22, 276, 248, 21, - /* 640 */ 80, 87, 79, 260, 290, 82, 241, 182, 182, 1, - /* 650 */ 2, 336, 34, 241, 255, 256, 37, 178, 179, 276, - /* 660 */ 145, 290, 102, 49, 349, 241, 276, 83, 353, 85, - /* 670 */ 86, 20, 88, 113, 118, 241, 92, 121, 122, 123, + /* 570 */ 194, 195, 12, 13, 18, 336, 20, 269, 291, 248, + /* 580 */ 20, 261, 22, 27, 139, 277, 30, 270, 349, 269, + /* 590 */ 92, 260, 353, 245, 92, 287, 248, 14, 278, 291, + /* 600 */ 20, 3, 22, 20, 48, 160, 251, 47, 277, 111, + /* 610 */ 112, 241, 114, 115, 116, 307, 241, 115, 310, 311, + /* 620 */ 312, 313, 314, 315, 64, 317, 241, 272, 320, 49, + /* 630 */ 145, 20, 324, 325, 326, 0, 14, 15, 16, 270, + /* 640 */ 80, 196, 93, 94, 95, 96, 97, 98, 99, 100, + /* 650 */ 101, 102, 103, 345, 105, 106, 107, 108, 109, 110, + /* 660 */ 270, 291, 102, 241, 241, 241, 291, 241, 241, 241, + /* 670 */ 241, 241, 241, 113, 118, 241, 291, 121, 122, 123, /* 680 */ 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, - /* 690 */ 134, 135, 136, 137, 138, 290, 20, 248, 248, 115, - /* 700 */ 248, 147, 290, 140, 314, 142, 0, 144, 248, 260, - /* 710 */ 260, 20, 260, 139, 290, 241, 156, 241, 241, 329, - /* 720 */ 330, 331, 41, 333, 290, 276, 276, 164, 276, 81, - /* 730 */ 241, 241, 241, 241, 160, 41, 276, 241, 178, 179, - /* 740 */ 20, 181, 182, 183, 184, 185, 186, 187, 188, 189, - /* 750 */ 190, 191, 192, 193, 194, 195, 248, 262, 269, 248, - /* 760 */ 265, 241, 241, 241, 290, 276, 290, 290, 260, 276, - /* 770 */ 196, 260, 255, 256, 314, 286, 283, 42, 43, 290, - /* 780 */ 290, 290, 290, 241, 276, 296, 290, 276, 286, 329, - /* 790 */ 330, 331, 0, 333, 292, 257, 307, 259, 92, 310, - /* 800 */ 311, 312, 313, 314, 315, 241, 317, 92, 41, 320, - /* 810 */ 290, 290, 290, 324, 325, 285, 286, 111, 112, 0, - /* 820 */ 114, 115, 116, 206, 207, 336, 207, 64, 84, 0, - /* 830 */ 115, 87, 290, 269, 167, 168, 145, 45, 349, 270, - /* 840 */ 276, 22, 353, 84, 18, 0, 87, 270, 81, 23, - /* 850 */ 286, 22, 84, 84, 290, 87, 87, 41, 182, 41, - /* 860 */ 296, 35, 36, 145, 146, 39, 241, 22, 41, 1, - /* 870 */ 2, 307, 193, 194, 310, 311, 312, 313, 314, 315, - /* 880 */ 4, 317, 56, 41, 320, 41, 270, 41, 324, 325, - /* 890 */ 41, 270, 80, 41, 269, 19, 270, 81, 299, 81, - /* 900 */ 336, 276, 90, 242, 41, 47, 80, 226, 81, 33, - /* 910 */ 258, 286, 41, 349, 47, 290, 0, 353, 356, 241, - /* 920 */ 41, 45, 228, 81, 341, 81, 50, 81, 347, 269, - /* 930 */ 81, 55, 307, 81, 303, 310, 311, 312, 313, 314, - /* 940 */ 315, 41, 317, 117, 81, 320, 41, 269, 247, 324, - /* 950 */ 325, 326, 81, 249, 276, 79, 279, 334, 82, 309, - /* 960 */ 81, 350, 350, 338, 286, 41, 337, 20, 290, 344, - /* 970 */ 345, 113, 241, 41, 148, 149, 150, 350, 41, 153, - /* 980 */ 113, 81, 248, 45, 158, 307, 81, 47, 310, 311, - /* 990 */ 312, 313, 314, 315, 304, 317, 170, 47, 255, 173, - /* 1000 */ 269, 175, 176, 177, 64, 81, 297, 276, 92, 154, - /* 1010 */ 248, 40, 248, 81, 284, 282, 139, 286, 81, 282, - /* 1020 */ 248, 290, 20, 243, 243, 20, 348, 111, 112, 301, - /* 1030 */ 114, 115, 116, 286, 208, 253, 253, 20, 307, 0, - /* 1040 */ 294, 310, 311, 312, 313, 314, 315, 241, 317, 253, - /* 1050 */ 276, 320, 20, 287, 253, 324, 325, 326, 253, 248, - /* 1060 */ 21, 253, 269, 24, 25, 26, 27, 28, 29, 30, - /* 1070 */ 31, 32, 269, 243, 243, 269, 345, 64, 269, 269, - /* 1080 */ 241, 248, 276, 269, 269, 290, 269, 301, 251, 269, - /* 1090 */ 269, 269, 286, 269, 163, 20, 290, 300, 251, 286, - /* 1100 */ 309, 294, 251, 216, 276, 251, 346, 215, 269, 223, - /* 1110 */ 287, 346, 222, 307, 343, 276, 310, 311, 312, 313, - /* 1120 */ 314, 315, 290, 317, 290, 286, 320, 291, 290, 290, - /* 1130 */ 324, 325, 326, 211, 291, 210, 342, 207, 340, 339, - /* 1140 */ 276, 335, 241, 20, 40, 227, 307, 230, 308, 310, - /* 1150 */ 311, 312, 313, 314, 315, 351, 317, 291, 225, 320, - /* 1160 */ 80, 291, 327, 324, 325, 326, 142, 276, 290, 265, - /* 1170 */ 269, 290, 290, 288, 335, 287, 80, 276, 251, 323, - /* 1180 */ 251, 272, 357, 276, 248, 259, 251, 286, 243, 0, - /* 1190 */ 295, 290, 298, 352, 252, 302, 241, 263, 239, 0, - /* 1200 */ 352, 351, 263, 72, 351, 241, 263, 0, 307, 352, - /* 1210 */ 47, 310, 311, 312, 313, 314, 315, 174, 317, 47, - /* 1220 */ 47, 320, 47, 174, 269, 324, 325, 326, 0, 47, - /* 1230 */ 47, 276, 0, 269, 174, 47, 335, 0, 47, 0, - /* 1240 */ 276, 286, 47, 0, 80, 290, 160, 159, 113, 156, - /* 1250 */ 286, 0, 0, 152, 290, 0, 151, 0, 44, 241, - /* 1260 */ 296, 0, 307, 0, 0, 310, 311, 312, 313, 314, - /* 1270 */ 315, 307, 317, 0, 310, 311, 312, 313, 314, 315, - /* 1280 */ 0, 317, 0, 0, 0, 0, 0, 269, 0, 0, - /* 1290 */ 0, 0, 0, 0, 276, 0, 0, 0, 40, 241, - /* 1300 */ 336, 0, 0, 0, 286, 0, 0, 0, 290, 22, - /* 1310 */ 355, 0, 0, 349, 0, 0, 0, 353, 241, 40, - /* 1320 */ 0, 44, 41, 45, 14, 307, 0, 269, 310, 311, - /* 1330 */ 312, 313, 314, 315, 276, 317, 37, 14, 320, 45, - /* 1340 */ 0, 37, 324, 325, 286, 0, 269, 0, 290, 0, - /* 1350 */ 0, 38, 44, 276, 37, 0, 0, 59, 37, 0, - /* 1360 */ 0, 37, 37, 286, 37, 307, 47, 290, 310, 311, - /* 1370 */ 312, 313, 314, 315, 241, 317, 47, 47, 320, 45, - /* 1380 */ 47, 45, 324, 325, 307, 0, 0, 310, 311, 312, - /* 1390 */ 313, 314, 315, 316, 317, 318, 319, 0, 0, 47, - /* 1400 */ 22, 0, 269, 19, 47, 47, 41, 89, 47, 276, - /* 1410 */ 47, 41, 0, 47, 47, 87, 22, 33, 47, 286, - /* 1420 */ 48, 22, 0, 290, 0, 22, 47, 0, 22, 45, - /* 1430 */ 0, 22, 0, 241, 20, 51, 52, 53, 54, 55, - /* 1440 */ 307, 0, 47, 310, 311, 312, 313, 314, 315, 0, - /* 1450 */ 317, 22, 0, 320, 0, 80, 37, 41, 325, 41, - /* 1460 */ 212, 269, 81, 79, 41, 44, 82, 145, 276, 81, - /* 1470 */ 80, 80, 80, 206, 81, 41, 145, 142, 286, 44, - /* 1480 */ 80, 140, 290, 41, 212, 293, 81, 241, 81, 44, - /* 1490 */ 44, 12, 13, 161, 41, 47, 81, 41, 81, 307, - /* 1500 */ 116, 22, 310, 311, 312, 313, 314, 315, 47, 317, - /* 1510 */ 47, 47, 212, 47, 47, 269, 2, 241, 41, 81, - /* 1520 */ 178, 81, 276, 80, 44, 141, 47, 80, 144, 44, - /* 1530 */ 80, 22, 286, 180, 0, 80, 290, 37, 81, 81, - /* 1540 */ 80, 90, 80, 64, 22, 269, 162, 143, 164, 80, - /* 1550 */ 241, 140, 276, 307, 80, 44, 310, 311, 312, 313, - /* 1560 */ 314, 315, 286, 317, 44, 319, 290, 80, 80, 293, - /* 1570 */ 81, 80, 91, 81, 47, 47, 80, 47, 269, 81, - /* 1580 */ 241, 102, 80, 307, 81, 276, 310, 311, 312, 313, - /* 1590 */ 314, 315, 113, 317, 47, 286, 81, 80, 47, 290, - /* 1600 */ 81, 80, 293, 47, 80, 22, 47, 113, 269, 104, - /* 1610 */ 241, 104, 248, 80, 104, 276, 307, 104, 92, 310, - /* 1620 */ 311, 312, 313, 314, 315, 286, 317, 47, 80, 290, - /* 1630 */ 80, 22, 59, 58, 64, 156, 78, 41, 269, 47, - /* 1640 */ 276, 47, 47, 22, 47, 276, 307, 47, 47, 310, - /* 1650 */ 311, 312, 313, 314, 315, 286, 317, 178, 47, 290, - /* 1660 */ 296, 241, 64, 47, 47, 47, 47, 47, 189, 190, - /* 1670 */ 191, 47, 47, 0, 47, 45, 307, 241, 314, 310, - /* 1680 */ 311, 312, 313, 314, 315, 37, 317, 0, 47, 269, - /* 1690 */ 45, 37, 0, 329, 330, 331, 276, 333, 47, 37, - /* 1700 */ 336, 0, 45, 45, 37, 269, 286, 47, 0, 46, - /* 1710 */ 290, 47, 276, 349, 0, 0, 22, 353, 21, 358, - /* 1720 */ 22, 22, 286, 21, 241, 20, 290, 307, 358, 358, - /* 1730 */ 310, 311, 312, 313, 314, 315, 358, 317, 358, 358, - /* 1740 */ 358, 358, 241, 307, 358, 358, 310, 311, 312, 313, - /* 1750 */ 314, 315, 269, 317, 358, 358, 358, 241, 358, 276, - /* 1760 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 286, - /* 1770 */ 269, 358, 358, 290, 358, 358, 358, 276, 358, 358, - /* 1780 */ 358, 358, 358, 358, 358, 269, 358, 286, 358, 358, - /* 1790 */ 307, 290, 276, 310, 311, 312, 313, 314, 315, 358, - /* 1800 */ 317, 358, 286, 358, 241, 358, 290, 358, 307, 358, - /* 1810 */ 358, 310, 311, 312, 313, 314, 315, 358, 317, 358, - /* 1820 */ 241, 358, 358, 307, 358, 358, 310, 311, 312, 313, - /* 1830 */ 314, 315, 269, 317, 358, 358, 358, 358, 358, 276, - /* 1840 */ 358, 358, 358, 358, 358, 358, 358, 358, 269, 286, - /* 1850 */ 358, 358, 358, 290, 358, 276, 358, 358, 358, 358, - /* 1860 */ 358, 358, 358, 358, 358, 286, 358, 241, 358, 290, - /* 1870 */ 307, 358, 358, 310, 311, 312, 313, 314, 315, 358, - /* 1880 */ 317, 358, 358, 358, 358, 358, 307, 358, 358, 310, - /* 1890 */ 311, 312, 313, 314, 315, 269, 317, 241, 358, 358, - /* 1900 */ 358, 358, 276, 358, 358, 358, 358, 358, 358, 358, - /* 1910 */ 358, 358, 286, 358, 358, 358, 290, 358, 358, 358, - /* 1920 */ 358, 358, 358, 358, 358, 269, 358, 358, 358, 358, - /* 1930 */ 358, 358, 276, 307, 358, 358, 310, 311, 312, 313, - /* 1940 */ 314, 315, 286, 317, 358, 358, 290, 241, 358, 358, - /* 1950 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 1960 */ 358, 358, 358, 307, 358, 358, 310, 311, 312, 313, - /* 1970 */ 314, 315, 358, 317, 358, 269, 358, 241, 358, 358, - /* 1980 */ 358, 358, 276, 358, 358, 358, 358, 358, 358, 358, - /* 1990 */ 358, 358, 286, 358, 358, 358, 290, 358, 358, 358, - /* 2000 */ 358, 358, 358, 358, 358, 269, 358, 358, 358, 358, - /* 2010 */ 358, 358, 276, 307, 358, 358, 310, 311, 312, 313, - /* 2020 */ 314, 315, 286, 317, 358, 358, 290, 241, 358, 358, - /* 2030 */ 358, 358, 358, 358, 358, 358, 241, 358, 358, 358, - /* 2040 */ 358, 358, 358, 307, 358, 358, 310, 311, 312, 313, - /* 2050 */ 314, 315, 358, 317, 358, 269, 358, 358, 358, 358, - /* 2060 */ 358, 358, 276, 358, 269, 358, 358, 358, 358, 358, - /* 2070 */ 358, 276, 286, 358, 358, 358, 290, 358, 358, 358, - /* 2080 */ 358, 286, 358, 358, 358, 290, 358, 358, 358, 358, - /* 2090 */ 241, 358, 358, 307, 358, 358, 310, 311, 312, 313, - /* 2100 */ 314, 315, 307, 317, 358, 310, 311, 312, 313, 314, - /* 2110 */ 315, 358, 317, 358, 358, 358, 358, 358, 269, 358, - /* 2120 */ 358, 358, 358, 358, 358, 276, 358, 358, 358, 358, - /* 2130 */ 358, 358, 358, 358, 358, 286, 358, 358, 358, 290, - /* 2140 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, - /* 2150 */ 358, 358, 358, 358, 358, 358, 307, 358, 358, 310, - /* 2160 */ 311, 312, 313, 314, 315, 358, 317, + /* 690 */ 134, 135, 136, 137, 138, 60, 61, 62, 63, 182, + /* 700 */ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + /* 710 */ 75, 76, 77, 291, 291, 291, 156, 291, 291, 291, + /* 720 */ 291, 291, 291, 241, 37, 291, 269, 241, 255, 256, + /* 730 */ 241, 255, 256, 286, 287, 278, 42, 43, 178, 179, + /* 740 */ 308, 181, 182, 183, 184, 185, 186, 187, 188, 189, + /* 750 */ 190, 191, 192, 193, 194, 195, 145, 18, 269, 248, + /* 760 */ 4, 241, 23, 308, 332, 182, 277, 47, 22, 248, + /* 770 */ 20, 260, 0, 291, 35, 36, 287, 291, 39, 287, + /* 780 */ 291, 260, 206, 207, 64, 293, 297, 332, 277, 269, + /* 790 */ 167, 168, 41, 47, 248, 56, 307, 277, 277, 310, + /* 800 */ 311, 312, 313, 314, 315, 262, 317, 287, 265, 320, + /* 810 */ 64, 291, 248, 324, 325, 41, 300, 45, 84, 80, + /* 820 */ 47, 87, 224, 277, 260, 336, 84, 307, 41, 87, + /* 830 */ 310, 311, 312, 313, 314, 315, 44, 317, 349, 248, + /* 840 */ 320, 277, 353, 297, 324, 325, 326, 84, 102, 84, + /* 850 */ 87, 260, 87, 21, 58, 81, 117, 241, 338, 113, + /* 860 */ 314, 248, 248, 270, 344, 345, 34, 47, 277, 145, + /* 870 */ 146, 41, 80, 260, 260, 329, 330, 331, 0, 333, + /* 880 */ 1, 2, 336, 0, 64, 269, 113, 148, 149, 150, + /* 890 */ 277, 277, 153, 277, 207, 349, 242, 158, 41, 353, + /* 900 */ 22, 47, 156, 287, 41, 22, 41, 291, 258, 170, + /* 910 */ 41, 81, 173, 41, 175, 176, 177, 41, 193, 194, + /* 920 */ 241, 0, 41, 307, 178, 179, 310, 311, 312, 313, + /* 930 */ 314, 315, 182, 317, 356, 41, 320, 347, 81, 80, + /* 940 */ 324, 325, 326, 304, 81, 41, 81, 208, 269, 90, + /* 950 */ 81, 335, 269, 81, 247, 41, 277, 81, 41, 341, + /* 960 */ 249, 33, 81, 334, 280, 209, 287, 113, 309, 350, + /* 970 */ 291, 41, 241, 45, 178, 81, 350, 226, 350, 51, + /* 980 */ 52, 53, 54, 55, 41, 81, 307, 41, 41, 310, + /* 990 */ 311, 312, 313, 314, 315, 81, 317, 0, 81, 320, + /* 1000 */ 269, 337, 20, 324, 325, 326, 248, 79, 277, 45, + /* 1010 */ 82, 81, 47, 92, 335, 228, 255, 305, 287, 298, + /* 1020 */ 154, 40, 291, 248, 81, 248, 285, 81, 81, 139, + /* 1030 */ 283, 241, 111, 112, 248, 114, 115, 116, 307, 20, + /* 1040 */ 283, 310, 311, 312, 313, 314, 315, 4, 317, 243, + /* 1050 */ 243, 320, 20, 253, 302, 324, 325, 326, 287, 269, + /* 1060 */ 20, 253, 19, 295, 253, 277, 335, 277, 140, 20, + /* 1070 */ 142, 288, 144, 248, 253, 253, 33, 287, 243, 253, + /* 1080 */ 64, 291, 241, 269, 269, 269, 269, 297, 45, 92, + /* 1090 */ 248, 302, 164, 50, 243, 291, 269, 307, 55, 241, + /* 1100 */ 310, 311, 312, 313, 314, 315, 269, 317, 111, 112, + /* 1110 */ 269, 114, 115, 116, 269, 269, 269, 269, 277, 269, + /* 1120 */ 163, 251, 79, 287, 301, 82, 336, 269, 287, 295, + /* 1130 */ 277, 251, 291, 251, 20, 277, 251, 288, 297, 349, + /* 1140 */ 309, 216, 215, 353, 346, 287, 223, 292, 307, 291, + /* 1150 */ 211, 310, 311, 312, 313, 314, 315, 241, 317, 291, + /* 1160 */ 222, 210, 343, 292, 346, 307, 291, 339, 310, 311, + /* 1170 */ 312, 313, 314, 315, 207, 317, 277, 336, 320, 291, + /* 1180 */ 342, 20, 324, 325, 0, 269, 40, 327, 227, 241, + /* 1190 */ 349, 352, 340, 277, 353, 308, 230, 225, 80, 292, + /* 1200 */ 357, 291, 291, 287, 292, 142, 291, 291, 24, 25, + /* 1210 */ 26, 27, 28, 29, 30, 31, 32, 269, 351, 248, + /* 1220 */ 323, 289, 277, 307, 351, 277, 310, 311, 312, 313, + /* 1230 */ 314, 315, 288, 317, 251, 287, 320, 251, 352, 291, + /* 1240 */ 324, 325, 241, 352, 351, 265, 277, 80, 277, 273, + /* 1250 */ 259, 248, 243, 251, 299, 307, 303, 263, 310, 311, + /* 1260 */ 312, 313, 314, 315, 316, 317, 318, 319, 297, 296, + /* 1270 */ 269, 263, 252, 263, 239, 0, 0, 72, 277, 0, + /* 1280 */ 47, 174, 47, 47, 47, 314, 174, 0, 287, 47, + /* 1290 */ 47, 174, 291, 0, 47, 0, 47, 0, 47, 241, + /* 1300 */ 329, 330, 331, 0, 333, 80, 159, 336, 307, 160, + /* 1310 */ 113, 310, 311, 312, 313, 314, 315, 0, 317, 156, + /* 1320 */ 349, 320, 0, 152, 353, 151, 325, 269, 19, 0, + /* 1330 */ 0, 0, 44, 0, 0, 277, 0, 0, 0, 0, + /* 1340 */ 0, 0, 33, 22, 40, 287, 241, 0, 0, 291, + /* 1350 */ 0, 0, 294, 0, 45, 0, 0, 0, 0, 0, + /* 1360 */ 51, 52, 53, 54, 55, 307, 0, 0, 310, 311, + /* 1370 */ 312, 313, 314, 315, 269, 317, 0, 0, 0, 0, + /* 1380 */ 0, 0, 277, 0, 0, 241, 0, 41, 79, 40, + /* 1390 */ 0, 82, 287, 0, 0, 0, 291, 37, 44, 14, + /* 1400 */ 38, 14, 37, 0, 0, 44, 37, 0, 0, 45, + /* 1410 */ 47, 59, 307, 269, 0, 310, 311, 312, 313, 314, + /* 1420 */ 315, 277, 317, 0, 0, 116, 37, 0, 37, 47, + /* 1430 */ 0, 287, 87, 45, 37, 291, 241, 37, 294, 47, + /* 1440 */ 0, 45, 0, 0, 45, 47, 0, 22, 47, 47, + /* 1450 */ 141, 307, 47, 144, 310, 311, 312, 313, 314, 315, + /* 1460 */ 355, 317, 89, 47, 269, 241, 47, 41, 0, 41, + /* 1470 */ 22, 162, 277, 164, 22, 0, 47, 22, 47, 0, + /* 1480 */ 47, 47, 287, 48, 22, 0, 291, 22, 0, 20, + /* 1490 */ 0, 47, 22, 269, 0, 0, 161, 241, 0, 41, + /* 1500 */ 80, 277, 307, 37, 41, 310, 311, 312, 313, 314, + /* 1510 */ 315, 287, 317, 145, 319, 291, 145, 142, 294, 212, + /* 1520 */ 81, 41, 81, 41, 206, 269, 81, 80, 44, 80, + /* 1530 */ 80, 307, 41, 277, 310, 311, 312, 313, 314, 315, + /* 1540 */ 44, 317, 81, 287, 80, 44, 140, 291, 81, 41, + /* 1550 */ 294, 44, 81, 241, 212, 41, 212, 81, 47, 2, + /* 1560 */ 47, 41, 47, 307, 47, 47, 310, 311, 312, 313, + /* 1570 */ 314, 315, 47, 317, 81, 44, 44, 80, 22, 143, + /* 1580 */ 81, 269, 80, 80, 241, 80, 80, 178, 0, 277, + /* 1590 */ 180, 81, 37, 140, 90, 44, 44, 81, 22, 287, + /* 1600 */ 81, 80, 80, 291, 80, 47, 80, 80, 80, 47, + /* 1610 */ 81, 81, 269, 241, 91, 80, 47, 80, 47, 307, + /* 1620 */ 277, 81, 310, 311, 312, 313, 314, 315, 80, 317, + /* 1630 */ 287, 81, 47, 80, 291, 81, 47, 104, 104, 80, + /* 1640 */ 104, 269, 104, 22, 80, 80, 47, 80, 47, 277, + /* 1650 */ 307, 22, 92, 310, 311, 312, 313, 314, 315, 287, + /* 1660 */ 317, 59, 241, 291, 58, 78, 41, 64, 12, 13, + /* 1670 */ 47, 47, 22, 241, 113, 47, 64, 47, 22, 307, + /* 1680 */ 47, 47, 310, 311, 312, 313, 314, 315, 47, 317, + /* 1690 */ 269, 47, 47, 47, 47, 47, 47, 0, 277, 47, + /* 1700 */ 47, 269, 45, 47, 37, 0, 47, 45, 287, 277, + /* 1710 */ 37, 0, 291, 47, 45, 37, 0, 47, 37, 287, + /* 1720 */ 64, 45, 0, 291, 241, 0, 47, 0, 307, 46, + /* 1730 */ 22, 310, 311, 312, 313, 314, 315, 22, 317, 307, + /* 1740 */ 21, 20, 310, 311, 312, 313, 314, 315, 22, 317, + /* 1750 */ 21, 358, 269, 241, 358, 358, 358, 358, 102, 358, + /* 1760 */ 277, 358, 358, 358, 358, 358, 358, 358, 358, 113, + /* 1770 */ 287, 358, 358, 358, 291, 358, 358, 358, 358, 358, + /* 1780 */ 358, 269, 358, 358, 358, 241, 358, 358, 358, 277, + /* 1790 */ 307, 358, 358, 310, 311, 312, 313, 314, 315, 287, + /* 1800 */ 317, 358, 358, 291, 358, 358, 358, 358, 358, 358, + /* 1810 */ 358, 358, 156, 269, 358, 358, 358, 358, 358, 307, + /* 1820 */ 358, 277, 310, 311, 312, 313, 314, 315, 358, 317, + /* 1830 */ 358, 287, 358, 358, 178, 291, 358, 358, 358, 358, + /* 1840 */ 358, 241, 358, 358, 358, 189, 190, 191, 358, 358, + /* 1850 */ 358, 307, 358, 358, 310, 311, 312, 313, 314, 315, + /* 1860 */ 358, 317, 358, 358, 358, 358, 358, 358, 358, 269, + /* 1870 */ 358, 358, 241, 358, 358, 358, 358, 277, 358, 358, + /* 1880 */ 358, 358, 358, 358, 358, 358, 358, 287, 358, 358, + /* 1890 */ 358, 291, 358, 358, 358, 358, 358, 358, 358, 358, + /* 1900 */ 269, 241, 358, 358, 358, 358, 358, 307, 277, 358, + /* 1910 */ 310, 311, 312, 313, 314, 315, 358, 317, 287, 358, + /* 1920 */ 358, 358, 291, 358, 358, 358, 358, 358, 358, 269, + /* 1930 */ 358, 358, 241, 358, 358, 358, 358, 277, 307, 358, + /* 1940 */ 358, 310, 311, 312, 313, 314, 315, 287, 317, 358, + /* 1950 */ 358, 291, 358, 358, 358, 358, 358, 358, 358, 358, + /* 1960 */ 269, 241, 358, 358, 358, 358, 358, 307, 277, 358, + /* 1970 */ 310, 311, 312, 313, 314, 315, 358, 317, 287, 358, + /* 1980 */ 358, 358, 291, 358, 358, 358, 358, 358, 358, 269, + /* 1990 */ 358, 358, 358, 358, 358, 358, 358, 277, 307, 358, + /* 2000 */ 358, 310, 311, 312, 313, 314, 315, 287, 317, 358, + /* 2010 */ 358, 291, 241, 358, 358, 358, 358, 358, 358, 358, + /* 2020 */ 358, 358, 358, 241, 358, 358, 358, 307, 358, 358, + /* 2030 */ 310, 311, 312, 313, 314, 315, 358, 317, 358, 358, + /* 2040 */ 269, 358, 358, 358, 358, 358, 358, 358, 277, 358, + /* 2050 */ 358, 269, 358, 358, 358, 358, 358, 358, 287, 277, + /* 2060 */ 358, 358, 291, 358, 358, 358, 358, 358, 358, 287, + /* 2070 */ 358, 358, 358, 291, 358, 358, 358, 241, 307, 358, + /* 2080 */ 358, 310, 311, 312, 313, 314, 315, 358, 317, 307, + /* 2090 */ 358, 358, 310, 311, 312, 313, 314, 315, 358, 317, + /* 2100 */ 358, 358, 358, 358, 358, 269, 358, 358, 358, 358, + /* 2110 */ 358, 358, 358, 277, 358, 358, 358, 358, 358, 358, + /* 2120 */ 358, 358, 358, 287, 358, 358, 358, 291, 358, 358, + /* 2130 */ 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, + /* 2140 */ 358, 358, 358, 307, 358, 358, 310, 311, 312, 313, + /* 2150 */ 314, 315, 358, 317, }; #define YY_SHIFT_COUNT (604) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1715) +#define YY_SHIFT_MAX (1729) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 826, 0, 0, 48, 96, 96, 96, 96, 280, 280, + /* 0 */ 739, 0, 0, 48, 96, 96, 96, 96, 280, 280, /* 10 */ 96, 96, 328, 376, 560, 376, 376, 376, 376, 376, /* 20 */ 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, - /* 30 */ 376, 376, 376, 376, 376, 376, 376, 376, 95, 95, - /* 40 */ 375, 375, 375, 1479, 1479, 1479, 100, 50, 171, 45, - /* 50 */ 45, 342, 342, 246, 171, 171, 45, 45, 45, 45, - /* 60 */ 45, 45, 17, 45, 201, 323, 651, 201, 45, 45, - /* 70 */ 201, 45, 201, 201, 651, 201, 45, 324, 556, 63, - /* 80 */ 14, 14, 13, 479, 422, 479, 479, 479, 479, 479, - /* 90 */ 479, 479, 479, 479, 479, 479, 479, 479, 479, 479, - /* 100 */ 479, 479, 479, 479, 584, 614, 524, 524, 301, 281, - /* 110 */ 432, 432, 432, 447, 281, 720, 651, 201, 201, 651, - /* 120 */ 348, 763, 319, 319, 319, 319, 319, 319, 319, 1384, - /* 130 */ 1039, 377, 349, 28, 104, 230, 465, 466, 735, 102, - /* 140 */ 715, 691, 617, 619, 617, 396, 396, 396, 268, 676, - /* 150 */ 947, 938, 950, 855, 947, 947, 971, 877, 877, 947, - /* 160 */ 1002, 1002, 1005, 17, 651, 17, 1017, 17, 720, 1032, - /* 170 */ 17, 17, 947, 17, 1002, 201, 201, 201, 201, 201, - /* 180 */ 201, 201, 201, 201, 201, 201, 947, 1002, 1013, 1005, - /* 190 */ 324, 931, 651, 324, 1017, 324, 720, 1032, 324, 1075, - /* 200 */ 887, 892, 1013, 887, 892, 1013, 1013, 201, 886, 890, - /* 210 */ 922, 925, 930, 720, 1123, 1104, 918, 933, 917, 918, - /* 220 */ 933, 918, 933, 1080, 892, 1013, 1013, 892, 1013, 1024, - /* 230 */ 720, 1032, 324, 348, 324, 720, 1096, 763, 947, 324, - /* 240 */ 1002, 2167, 2167, 2167, 2167, 2167, 2167, 2167, 2167, 244, - /* 250 */ 563, 141, 876, 706, 916, 241, 84, 355, 515, 419, - /* 260 */ 85, 282, 282, 282, 282, 282, 282, 282, 282, 239, - /* 270 */ 469, 470, 554, 648, 574, 36, 36, 36, 36, 792, - /* 280 */ 767, 744, 759, 768, 769, 819, 829, 845, 618, 667, - /* 290 */ 718, 816, 818, 827, 868, 679, 681, 694, 842, 145, - /* 300 */ 844, 455, 846, 849, 852, 863, 871, 858, 867, 879, - /* 310 */ 900, 905, 924, 932, 937, 812, 940, 1189, 1199, 1131, - /* 320 */ 1207, 1163, 1043, 1172, 1173, 1175, 1049, 1228, 1182, 1183, - /* 330 */ 1060, 1232, 1188, 1237, 1191, 1239, 1195, 1243, 1164, 1086, - /* 340 */ 1088, 1135, 1093, 1251, 1252, 1101, 1105, 1255, 1257, 1214, - /* 350 */ 1261, 1263, 1264, 1273, 1280, 1282, 1283, 1284, 1285, 1286, - /* 360 */ 1288, 1289, 1290, 1291, 1292, 1293, 1295, 1296, 1258, 1297, - /* 370 */ 1301, 1302, 1303, 1305, 1306, 1287, 1307, 1311, 1312, 1314, - /* 380 */ 1315, 1316, 1279, 1299, 1281, 1310, 1277, 1323, 1308, 1320, - /* 390 */ 1313, 1304, 1326, 1345, 1347, 1349, 1317, 1350, 1298, 1355, - /* 400 */ 1356, 1319, 1278, 1321, 1359, 1329, 1294, 1324, 1340, 1330, - /* 410 */ 1334, 1325, 1360, 1333, 1336, 1327, 1385, 1386, 1397, 1398, - /* 420 */ 1318, 1328, 1352, 1378, 1401, 1357, 1358, 1361, 1363, 1365, - /* 430 */ 1370, 1366, 1367, 1371, 1412, 1394, 1424, 1399, 1372, 1422, - /* 440 */ 1403, 1379, 1427, 1406, 1430, 1409, 1414, 1432, 1322, 1395, - /* 450 */ 1441, 1332, 1429, 1331, 1335, 1449, 1452, 1454, 1375, 1419, - /* 460 */ 1341, 1416, 1418, 1248, 1381, 1423, 1388, 1390, 1391, 1392, - /* 470 */ 1393, 1434, 1421, 1435, 1400, 1442, 1272, 1405, 1407, 1445, - /* 480 */ 1267, 1453, 1446, 1415, 1456, 1300, 1417, 1448, 1461, 1463, - /* 490 */ 1464, 1466, 1467, 1417, 1514, 1342, 1477, 1438, 1443, 1440, - /* 500 */ 1480, 1447, 1450, 1485, 1509, 1353, 1455, 1457, 1458, 1460, - /* 510 */ 1462, 1404, 1469, 1534, 1500, 1411, 1474, 1451, 1511, 1520, - /* 520 */ 1487, 1489, 1488, 1522, 1491, 1481, 1492, 1527, 1528, 1496, - /* 530 */ 1498, 1530, 1502, 1503, 1547, 1517, 1515, 1551, 1521, 1519, - /* 540 */ 1556, 1524, 1505, 1507, 1510, 1513, 1583, 1526, 1533, 1548, - /* 550 */ 1559, 1550, 1494, 1580, 1609, 1573, 1575, 1570, 1558, 1596, - /* 560 */ 1592, 1594, 1595, 1597, 1600, 1621, 1601, 1611, 1598, 1365, - /* 570 */ 1616, 1370, 1617, 1618, 1619, 1620, 1624, 1625, 1673, 1627, - /* 580 */ 1630, 1648, 1687, 1641, 1645, 1654, 1692, 1651, 1657, 1662, - /* 590 */ 1701, 1660, 1658, 1667, 1708, 1664, 1663, 1714, 1715, 1694, - /* 600 */ 1697, 1698, 1699, 1702, 1705, + /* 30 */ 376, 376, 376, 376, 376, 376, 376, 376, 37, 37, + /* 40 */ 68, 68, 68, 1656, 1656, 1656, 1656, 64, 271, 179, + /* 50 */ 18, 18, 13, 13, 123, 179, 179, 18, 18, 18, + /* 60 */ 18, 18, 18, 35, 18, 182, 190, 314, 182, 18, + /* 70 */ 18, 182, 18, 182, 182, 314, 182, 18, 334, 556, + /* 80 */ 343, 107, 107, 192, 312, 746, 746, 746, 746, 746, + /* 90 */ 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, + /* 100 */ 746, 746, 746, 746, 441, 580, 110, 110, 206, 720, + /* 110 */ 393, 393, 393, 244, 720, 384, 314, 182, 182, 314, + /* 120 */ 356, 466, 549, 549, 549, 549, 549, 549, 549, 1309, + /* 130 */ 294, 197, 21, 78, 268, 279, 517, 583, 694, 292, + /* 140 */ 502, 611, 576, 687, 576, 598, 598, 598, 756, 750, + /* 150 */ 982, 964, 965, 866, 982, 982, 981, 890, 890, 982, + /* 160 */ 1019, 1019, 1032, 35, 314, 35, 1040, 35, 384, 1049, + /* 170 */ 35, 35, 982, 35, 1019, 182, 182, 182, 182, 182, + /* 180 */ 182, 182, 182, 182, 182, 182, 982, 1019, 1016, 1032, + /* 190 */ 334, 957, 314, 334, 1040, 334, 384, 1049, 334, 1114, + /* 200 */ 925, 927, 1016, 925, 927, 1016, 1016, 182, 923, 938, + /* 210 */ 939, 951, 967, 384, 1161, 1146, 961, 972, 966, 961, + /* 220 */ 972, 961, 972, 1118, 927, 1016, 1016, 927, 1016, 1063, + /* 230 */ 384, 1049, 334, 356, 334, 384, 1167, 466, 982, 334, + /* 240 */ 1019, 2154, 2154, 2154, 2154, 2154, 2154, 2154, 2154, 635, + /* 250 */ 928, 1184, 1043, 921, 997, 119, 14, 30, 485, 126, + /* 260 */ 498, 354, 354, 354, 354, 354, 354, 354, 354, 309, + /* 270 */ 321, 415, 404, 8, 445, 622, 622, 622, 622, 772, + /* 280 */ 774, 734, 742, 763, 765, 435, 878, 883, 832, 623, + /* 290 */ 724, 830, 857, 863, 879, 725, 751, 787, 865, 796, + /* 300 */ 869, 792, 872, 876, 881, 894, 904, 773, 854, 914, + /* 310 */ 917, 930, 943, 946, 947, 859, 820, 1275, 1276, 1205, + /* 320 */ 1279, 1233, 1107, 1235, 1236, 1237, 1112, 1287, 1242, 1243, + /* 330 */ 1117, 1293, 1247, 1295, 1249, 1297, 1251, 1303, 1225, 1149, + /* 340 */ 1147, 1197, 1163, 1317, 1322, 1171, 1174, 1329, 1330, 1288, + /* 350 */ 1331, 1333, 1334, 1336, 1337, 1338, 1339, 1340, 1341, 1347, + /* 360 */ 1348, 1350, 1351, 1353, 1355, 1356, 1357, 1358, 1304, 1359, + /* 370 */ 1366, 1367, 1376, 1377, 1378, 1321, 1379, 1380, 1381, 1383, + /* 380 */ 1384, 1386, 1349, 1360, 1346, 1385, 1354, 1387, 1361, 1390, + /* 390 */ 1362, 1365, 1393, 1394, 1395, 1403, 1369, 1404, 1352, 1407, + /* 400 */ 1408, 1363, 1364, 1389, 1414, 1382, 1388, 1391, 1423, 1392, + /* 410 */ 1396, 1397, 1424, 1398, 1399, 1400, 1427, 1430, 1440, 1442, + /* 420 */ 1373, 1345, 1401, 1425, 1443, 1402, 1405, 1416, 1419, 1426, + /* 430 */ 1428, 1429, 1431, 1433, 1446, 1448, 1468, 1452, 1435, 1475, + /* 440 */ 1455, 1434, 1479, 1462, 1485, 1465, 1469, 1488, 1368, 1444, + /* 450 */ 1490, 1335, 1470, 1371, 1375, 1494, 1495, 1498, 1420, 1466, + /* 460 */ 1406, 1458, 1463, 1307, 1439, 1480, 1441, 1447, 1449, 1450, + /* 470 */ 1445, 1482, 1484, 1496, 1464, 1491, 1342, 1461, 1467, 1501, + /* 480 */ 1318, 1508, 1507, 1471, 1514, 1344, 1476, 1511, 1513, 1515, + /* 490 */ 1517, 1518, 1525, 1476, 1557, 1409, 1520, 1493, 1497, 1499, + /* 500 */ 1531, 1502, 1503, 1532, 1556, 1410, 1505, 1510, 1516, 1506, + /* 510 */ 1521, 1436, 1522, 1588, 1555, 1453, 1524, 1504, 1551, 1552, + /* 520 */ 1526, 1519, 1527, 1576, 1528, 1523, 1529, 1558, 1562, 1535, + /* 530 */ 1530, 1569, 1537, 1540, 1571, 1548, 1550, 1585, 1553, 1554, + /* 540 */ 1589, 1559, 1533, 1534, 1536, 1538, 1621, 1560, 1564, 1565, + /* 550 */ 1599, 1567, 1561, 1601, 1629, 1602, 1606, 1603, 1587, 1625, + /* 560 */ 1623, 1624, 1628, 1630, 1633, 1650, 1634, 1641, 1612, 1426, + /* 570 */ 1644, 1428, 1645, 1646, 1647, 1648, 1649, 1652, 1697, 1653, + /* 580 */ 1657, 1667, 1705, 1659, 1662, 1673, 1711, 1666, 1669, 1678, + /* 590 */ 1716, 1670, 1676, 1681, 1722, 1679, 1683, 1725, 1727, 1708, + /* 600 */ 1719, 1715, 1726, 1729, 1721, }; #define YY_REDUCE_COUNT (248) -#define YY_REDUCE_MIN (-317) -#define YY_REDUCE_MAX (1849) +#define YY_REDUCE_MIN (-335) +#define YY_REDUCE_MAX (1836) static const short yy_reduce_ofst[] = { - /* 0 */ 38, 489, 564, 625, 731, 806, 839, 901, 315, 964, - /* 10 */ 1018, 1058, 1077, -223, 1133, 90, 678, 955, 1192, 1246, - /* 20 */ 1276, 1309, 1339, 1369, 1420, 1436, 1483, 1501, 1516, 1563, - /* 30 */ 1579, 1626, 1656, 1706, 1736, 1786, 1795, 1849, -191, 1364, - /* 40 */ 23, 390, 460, -224, 235, -282, 257, -280, 197, -141, - /* 50 */ -112, -244, -240, -317, -202, -190, -243, 200, 327, 361, - /* 60 */ 383, 449, -152, 450, -221, 60, -64, -144, 337, 452, - /* 70 */ 55, 508, 78, 223, 47, 117, 511, -250, -177, -312, - /* 80 */ -312, -312, -113, 212, -34, 298, 308, 354, 371, 405, - /* 90 */ 412, 424, 434, 474, 476, 477, 490, 491, 492, 496, - /* 100 */ 520, 521, 522, 542, 70, -139, 161, 289, 326, 399, - /* 110 */ -277, 149, 291, -114, 517, 493, 502, 325, 267, 530, - /* 120 */ 495, 538, -259, -255, 569, 577, 616, 621, 626, 599, - /* 130 */ 661, 652, 562, 581, 631, 583, 660, 660, 701, 704, - /* 140 */ 677, 650, 623, 623, 623, 611, 612, 627, 629, 660, - /* 150 */ 734, 690, 743, 709, 762, 764, 730, 733, 737, 772, - /* 160 */ 780, 781, 728, 782, 747, 783, 746, 796, 774, 766, - /* 170 */ 801, 805, 811, 808, 830, 793, 803, 809, 810, 814, - /* 180 */ 815, 817, 820, 821, 822, 824, 833, 831, 795, 786, - /* 190 */ 837, 797, 813, 847, 807, 851, 828, 823, 854, 791, - /* 200 */ 760, 836, 832, 765, 843, 834, 838, 660, 771, 794, - /* 210 */ 798, 800, 623, 864, 840, 835, 841, 804, 825, 848, - /* 220 */ 850, 857, 853, 856, 866, 878, 881, 870, 882, 885, - /* 230 */ 891, 888, 927, 904, 929, 907, 909, 926, 936, 935, - /* 240 */ 945, 894, 893, 895, 934, 939, 943, 942, 959, + /* 0 */ -190, -238, 489, 520, 308, 616, 679, 731, 790, 841, + /* 10 */ 858, 916, 948, -140, 1001, 1058, 138, 1105, 1144, 1195, + /* 20 */ 1224, 1256, 1312, 1343, 1372, 1421, 1432, 1483, 1512, 1544, + /* 30 */ 1600, 1631, 1660, 1691, 1720, 1771, 1782, 1836, 546, 971, + /* 40 */ 88, -1, 113, -266, -181, 46, 103, 239, -286, -85, + /* 50 */ -163, 87, -240, -223, -335, -334, -253, -49, 85, 135, + /* 60 */ 248, 331, 511, -172, 521, -124, -111, 56, 136, 564, + /* 70 */ 591, -21, 613, 216, 130, 152, 320, 614, -173, -177, + /* 80 */ -256, -256, -256, -188, -105, -86, 213, 261, 287, 370, + /* 90 */ 375, 385, 422, 423, 424, 426, 427, 428, 429, 430, + /* 100 */ 431, 434, 482, 486, -100, 4, 305, 348, 63, 473, + /* 110 */ -83, 432, 455, 355, 476, 94, 492, 457, -8, 447, + /* 120 */ 543, -95, -270, 225, 235, 317, 369, 390, 593, 516, + /* 130 */ 654, 650, 578, 590, 639, 618, 683, 683, 707, 711, + /* 140 */ 684, 659, 629, 629, 629, 619, 626, 628, 664, 683, + /* 150 */ 758, 712, 761, 721, 775, 777, 741, 747, 757, 786, + /* 160 */ 806, 807, 752, 800, 771, 808, 768, 811, 788, 783, + /* 170 */ 821, 822, 825, 826, 835, 814, 815, 816, 817, 827, + /* 180 */ 837, 845, 846, 847, 848, 850, 842, 851, 804, 789, + /* 190 */ 870, 823, 836, 880, 834, 882, 853, 849, 885, 831, + /* 200 */ 798, 855, 868, 818, 871, 875, 888, 683, 819, 838, + /* 210 */ 852, 828, 629, 899, 887, 860, 839, 867, 843, 886, + /* 220 */ 873, 891, 893, 897, 907, 910, 911, 912, 915, 932, + /* 230 */ 945, 944, 983, 980, 986, 969, 976, 991, 1003, 1002, + /* 240 */ 1009, 955, 953, 973, 994, 1008, 1010, 1020, 1035, }; static const YYACTIONTYPE yy_default[] = { /* 0 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, @@ -753,9 +751,9 @@ static const YYACTIONTYPE yy_default[] = { /* 30 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, /* 40 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, /* 50 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, - /* 60 */ 1345, 1345, 1414, 1345, 1345, 1345, 1345, 1345, 1345, 1345, - /* 70 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1412, 1552, 1345, - /* 80 */ 1717, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 60 */ 1345, 1345, 1345, 1414, 1345, 1345, 1345, 1345, 1345, 1345, + /* 70 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1412, 1552, + /* 80 */ 1345, 1717, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, /* 90 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, /* 100 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1414, 1345, /* 110 */ 1728, 1728, 1728, 1412, 1345, 1345, 1345, 1345, 1345, 1345, @@ -1423,42 +1421,42 @@ static const char *const yyTokenName[] = { /* 268 */ "alter_table_options", /* 269 */ "column_name", /* 270 */ "type_name", - /* 271 */ "create_subtable_clause", - /* 272 */ "specific_tags_opt", - /* 273 */ "literal_list", - /* 274 */ "drop_table_clause", - /* 275 */ "col_name_list", - /* 276 */ "table_name", - /* 277 */ "column_def", - /* 278 */ "func_name_list", - /* 279 */ "alter_table_option", - /* 280 */ "col_name", - /* 281 */ "db_name_cond_opt", - /* 282 */ "like_pattern_opt", - /* 283 */ "table_name_cond", - /* 284 */ "from_db_opt", - /* 285 */ "func_name", - /* 286 */ "function_name", - /* 287 */ "index_name", - /* 288 */ "index_options", - /* 289 */ "func_list", - /* 290 */ "duration_literal", - /* 291 */ "sliding_opt", - /* 292 */ "func", - /* 293 */ "expression_list", - /* 294 */ "topic_name", - /* 295 */ "topic_options", - /* 296 */ "query_expression", - /* 297 */ "analyze_opt", - /* 298 */ "explain_options", - /* 299 */ "agg_func_opt", - /* 300 */ "bufsize_opt", - /* 301 */ "stream_name", - /* 302 */ "stream_options", - /* 303 */ "into_opt", - /* 304 */ "dnode_list", - /* 305 */ "signed", - /* 306 */ "signed_literal", + /* 271 */ "signed_literal", + /* 272 */ "create_subtable_clause", + /* 273 */ "specific_tags_opt", + /* 274 */ "literal_list", + /* 275 */ "drop_table_clause", + /* 276 */ "col_name_list", + /* 277 */ "table_name", + /* 278 */ "column_def", + /* 279 */ "func_name_list", + /* 280 */ "alter_table_option", + /* 281 */ "col_name", + /* 282 */ "db_name_cond_opt", + /* 283 */ "like_pattern_opt", + /* 284 */ "table_name_cond", + /* 285 */ "from_db_opt", + /* 286 */ "func_name", + /* 287 */ "function_name", + /* 288 */ "index_name", + /* 289 */ "index_options", + /* 290 */ "func_list", + /* 291 */ "duration_literal", + /* 292 */ "sliding_opt", + /* 293 */ "func", + /* 294 */ "expression_list", + /* 295 */ "topic_name", + /* 296 */ "topic_options", + /* 297 */ "query_expression", + /* 298 */ "analyze_opt", + /* 299 */ "explain_options", + /* 300 */ "agg_func_opt", + /* 301 */ "bufsize_opt", + /* 302 */ "stream_name", + /* 303 */ "stream_options", + /* 304 */ "into_opt", + /* 305 */ "dnode_list", + /* 306 */ "signed", /* 307 */ "literal_func", /* 308 */ "table_alias", /* 309 */ "column_alias", @@ -1638,7 +1636,7 @@ static const char *const yyRuleName[] = { /* 118 */ "alter_table_clause ::= full_table_name DROP TAG column_name", /* 119 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", /* 120 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", - /* 121 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal", + /* 121 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", /* 122 */ "multi_create_clause ::= create_subtable_clause", /* 123 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", /* 124 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options", @@ -2104,26 +2102,26 @@ static void yy_destructor( case 263: /* table_options */ case 267: /* alter_table_clause */ case 268: /* alter_table_options */ - case 271: /* create_subtable_clause */ - case 274: /* drop_table_clause */ - case 277: /* column_def */ - case 280: /* col_name */ - case 281: /* db_name_cond_opt */ - case 282: /* like_pattern_opt */ - case 283: /* table_name_cond */ - case 284: /* from_db_opt */ - case 285: /* func_name */ - case 288: /* index_options */ - case 290: /* duration_literal */ - case 291: /* sliding_opt */ - case 292: /* func */ - case 295: /* topic_options */ - case 296: /* query_expression */ - case 298: /* explain_options */ - case 302: /* stream_options */ - case 303: /* into_opt */ - case 305: /* signed */ - case 306: /* signed_literal */ + case 271: /* signed_literal */ + case 272: /* create_subtable_clause */ + case 275: /* drop_table_clause */ + case 278: /* column_def */ + case 281: /* col_name */ + case 282: /* db_name_cond_opt */ + case 283: /* like_pattern_opt */ + case 284: /* table_name_cond */ + case 285: /* from_db_opt */ + case 286: /* func_name */ + case 289: /* index_options */ + case 291: /* duration_literal */ + case 292: /* sliding_opt */ + case 293: /* func */ + case 296: /* topic_options */ + case 297: /* query_expression */ + case 299: /* explain_options */ + case 303: /* stream_options */ + case 304: /* into_opt */ + case 306: /* signed */ case 307: /* literal_func */ case 310: /* expression */ case 311: /* pseudo_column */ @@ -2161,7 +2159,7 @@ static void yy_destructor( case 239: /* account_options */ case 240: /* alter_account_options */ case 242: /* alter_account_option */ - case 300: /* bufsize_opt */ + case 301: /* bufsize_opt */ { } @@ -2172,11 +2170,11 @@ static void yy_destructor( case 249: /* dnode_endpoint */ case 250: /* dnode_host_name */ case 269: /* column_name */ - case 276: /* table_name */ - case 286: /* function_name */ - case 287: /* index_name */ - case 294: /* topic_name */ - case 301: /* stream_name */ + case 277: /* table_name */ + case 287: /* function_name */ + case 288: /* index_name */ + case 295: /* topic_name */ + case 302: /* stream_name */ case 308: /* table_alias */ case 309: /* column_alias */ case 315: /* star_func */ @@ -2195,8 +2193,8 @@ static void yy_destructor( break; case 251: /* not_exists_opt */ case 253: /* exists_opt */ - case 297: /* analyze_opt */ - case 299: /* agg_func_opt */ + case 298: /* analyze_opt */ + case 300: /* agg_func_opt */ case 337: /* set_quantifier_opt */ { @@ -2210,13 +2208,13 @@ static void yy_destructor( case 264: /* multi_create_clause */ case 265: /* tags_def */ case 266: /* multi_drop_clause */ - case 272: /* specific_tags_opt */ - case 273: /* literal_list */ - case 275: /* col_name_list */ - case 278: /* func_name_list */ - case 289: /* func_list */ - case 293: /* expression_list */ - case 304: /* dnode_list */ + case 273: /* specific_tags_opt */ + case 274: /* literal_list */ + case 276: /* col_name_list */ + case 279: /* func_name_list */ + case 290: /* func_list */ + case 294: /* expression_list */ + case 305: /* dnode_list */ case 316: /* star_func_para_list */ case 318: /* other_para_list */ case 338: /* select_list */ @@ -2231,7 +2229,7 @@ static void yy_destructor( } break; case 258: /* alter_db_option */ - case 279: /* alter_table_option */ + case 280: /* alter_table_option */ { } @@ -2682,21 +2680,21 @@ static const struct { { 267, -4 }, /* (118) alter_table_clause ::= full_table_name DROP TAG column_name */ { 267, -5 }, /* (119) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ { 267, -5 }, /* (120) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 267, -6 }, /* (121) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ + { 267, -6 }, /* (121) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ { 264, -1 }, /* (122) multi_create_clause ::= create_subtable_clause */ { 264, -2 }, /* (123) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 271, -10 }, /* (124) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ + { 272, -10 }, /* (124) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ { 266, -1 }, /* (125) multi_drop_clause ::= drop_table_clause */ { 266, -2 }, /* (126) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 274, -2 }, /* (127) drop_table_clause ::= exists_opt full_table_name */ - { 272, 0 }, /* (128) specific_tags_opt ::= */ - { 272, -3 }, /* (129) specific_tags_opt ::= NK_LP col_name_list NK_RP */ + { 275, -2 }, /* (127) drop_table_clause ::= exists_opt full_table_name */ + { 273, 0 }, /* (128) specific_tags_opt ::= */ + { 273, -3 }, /* (129) specific_tags_opt ::= NK_LP col_name_list NK_RP */ { 260, -1 }, /* (130) full_table_name ::= table_name */ { 260, -3 }, /* (131) full_table_name ::= db_name NK_DOT table_name */ { 261, -1 }, /* (132) column_def_list ::= column_def */ { 261, -3 }, /* (133) column_def_list ::= column_def_list NK_COMMA column_def */ - { 277, -2 }, /* (134) column_def ::= column_name type_name */ - { 277, -4 }, /* (135) column_def ::= column_name type_name COMMENT NK_STRING */ + { 278, -2 }, /* (134) column_def ::= column_name type_name */ + { 278, -4 }, /* (135) column_def ::= column_name type_name COMMENT NK_STRING */ { 270, -1 }, /* (136) type_name ::= BOOL */ { 270, -1 }, /* (137) type_name ::= TINYINT */ { 270, -1 }, /* (138) type_name ::= SMALLINT */ @@ -2732,11 +2730,11 @@ static const struct { { 263, -5 }, /* (168) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ { 268, -1 }, /* (169) alter_table_options ::= alter_table_option */ { 268, -2 }, /* (170) alter_table_options ::= alter_table_options alter_table_option */ - { 279, -2 }, /* (171) alter_table_option ::= COMMENT NK_STRING */ - { 279, -2 }, /* (172) alter_table_option ::= TTL NK_INTEGER */ - { 275, -1 }, /* (173) col_name_list ::= col_name */ - { 275, -3 }, /* (174) col_name_list ::= col_name_list NK_COMMA col_name */ - { 280, -1 }, /* (175) col_name ::= column_name */ + { 280, -2 }, /* (171) alter_table_option ::= COMMENT NK_STRING */ + { 280, -2 }, /* (172) alter_table_option ::= TTL NK_INTEGER */ + { 276, -1 }, /* (173) col_name_list ::= col_name */ + { 276, -3 }, /* (174) col_name_list ::= col_name_list NK_COMMA col_name */ + { 281, -1 }, /* (175) col_name ::= column_name */ { 238, -2 }, /* (176) cmd ::= SHOW DNODES */ { 238, -2 }, /* (177) cmd ::= SHOW USERS */ { 238, -2 }, /* (178) cmd ::= SHOW DATABASES */ @@ -2765,64 +2763,64 @@ static const struct { { 238, -2 }, /* (201) cmd ::= SHOW SNODES */ { 238, -2 }, /* (202) cmd ::= SHOW CLUSTER */ { 238, -2 }, /* (203) cmd ::= SHOW TRANSACTIONS */ - { 281, 0 }, /* (204) db_name_cond_opt ::= */ - { 281, -2 }, /* (205) db_name_cond_opt ::= db_name NK_DOT */ - { 282, 0 }, /* (206) like_pattern_opt ::= */ - { 282, -2 }, /* (207) like_pattern_opt ::= LIKE NK_STRING */ - { 283, -1 }, /* (208) table_name_cond ::= table_name */ - { 284, 0 }, /* (209) from_db_opt ::= */ - { 284, -2 }, /* (210) from_db_opt ::= FROM db_name */ - { 278, -1 }, /* (211) func_name_list ::= func_name */ - { 278, -3 }, /* (212) func_name_list ::= func_name_list NK_COMMA func_name */ - { 285, -1 }, /* (213) func_name ::= function_name */ + { 282, 0 }, /* (204) db_name_cond_opt ::= */ + { 282, -2 }, /* (205) db_name_cond_opt ::= db_name NK_DOT */ + { 283, 0 }, /* (206) like_pattern_opt ::= */ + { 283, -2 }, /* (207) like_pattern_opt ::= LIKE NK_STRING */ + { 284, -1 }, /* (208) table_name_cond ::= table_name */ + { 285, 0 }, /* (209) from_db_opt ::= */ + { 285, -2 }, /* (210) from_db_opt ::= FROM db_name */ + { 279, -1 }, /* (211) func_name_list ::= func_name */ + { 279, -3 }, /* (212) func_name_list ::= func_name_list NK_COMMA func_name */ + { 286, -1 }, /* (213) func_name ::= function_name */ { 238, -8 }, /* (214) cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ { 238, -10 }, /* (215) cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ { 238, -6 }, /* (216) cmd ::= DROP INDEX exists_opt index_name ON table_name */ - { 288, 0 }, /* (217) index_options ::= */ - { 288, -9 }, /* (218) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ - { 288, -11 }, /* (219) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ - { 289, -1 }, /* (220) func_list ::= func */ - { 289, -3 }, /* (221) func_list ::= func_list NK_COMMA func */ - { 292, -4 }, /* (222) func ::= function_name NK_LP expression_list NK_RP */ + { 289, 0 }, /* (217) index_options ::= */ + { 289, -9 }, /* (218) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ + { 289, -11 }, /* (219) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ + { 290, -1 }, /* (220) func_list ::= func */ + { 290, -3 }, /* (221) func_list ::= func_list NK_COMMA func */ + { 293, -4 }, /* (222) func ::= function_name NK_LP expression_list NK_RP */ { 238, -7 }, /* (223) cmd ::= CREATE TOPIC not_exists_opt topic_name topic_options AS query_expression */ { 238, -7 }, /* (224) cmd ::= CREATE TOPIC not_exists_opt topic_name topic_options AS db_name */ { 238, -4 }, /* (225) cmd ::= DROP TOPIC exists_opt topic_name */ - { 295, 0 }, /* (226) topic_options ::= */ - { 295, -3 }, /* (227) topic_options ::= topic_options WITH TABLE */ - { 295, -3 }, /* (228) topic_options ::= topic_options WITH SCHEMA */ - { 295, -3 }, /* (229) topic_options ::= topic_options WITH TAG */ + { 296, 0 }, /* (226) topic_options ::= */ + { 296, -3 }, /* (227) topic_options ::= topic_options WITH TABLE */ + { 296, -3 }, /* (228) topic_options ::= topic_options WITH SCHEMA */ + { 296, -3 }, /* (229) topic_options ::= topic_options WITH TAG */ { 238, -2 }, /* (230) cmd ::= DESC full_table_name */ { 238, -2 }, /* (231) cmd ::= DESCRIBE full_table_name */ { 238, -3 }, /* (232) cmd ::= RESET QUERY CACHE */ { 238, -4 }, /* (233) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ - { 297, 0 }, /* (234) analyze_opt ::= */ - { 297, -1 }, /* (235) analyze_opt ::= ANALYZE */ - { 298, 0 }, /* (236) explain_options ::= */ - { 298, -3 }, /* (237) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 298, -3 }, /* (238) explain_options ::= explain_options RATIO NK_FLOAT */ + { 298, 0 }, /* (234) analyze_opt ::= */ + { 298, -1 }, /* (235) analyze_opt ::= ANALYZE */ + { 299, 0 }, /* (236) explain_options ::= */ + { 299, -3 }, /* (237) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 299, -3 }, /* (238) explain_options ::= explain_options RATIO NK_FLOAT */ { 238, -6 }, /* (239) cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ { 238, -10 }, /* (240) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ { 238, -4 }, /* (241) cmd ::= DROP FUNCTION exists_opt function_name */ - { 299, 0 }, /* (242) agg_func_opt ::= */ - { 299, -1 }, /* (243) agg_func_opt ::= AGGREGATE */ - { 300, 0 }, /* (244) bufsize_opt ::= */ - { 300, -2 }, /* (245) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 300, 0 }, /* (242) agg_func_opt ::= */ + { 300, -1 }, /* (243) agg_func_opt ::= AGGREGATE */ + { 301, 0 }, /* (244) bufsize_opt ::= */ + { 301, -2 }, /* (245) bufsize_opt ::= BUFSIZE NK_INTEGER */ { 238, -8 }, /* (246) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ { 238, -4 }, /* (247) cmd ::= DROP STREAM exists_opt stream_name */ - { 303, 0 }, /* (248) into_opt ::= */ - { 303, -2 }, /* (249) into_opt ::= INTO full_table_name */ - { 302, 0 }, /* (250) stream_options ::= */ - { 302, -3 }, /* (251) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 302, -3 }, /* (252) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 302, -3 }, /* (253) stream_options ::= stream_options WATERMARK duration_literal */ + { 304, 0 }, /* (248) into_opt ::= */ + { 304, -2 }, /* (249) into_opt ::= INTO full_table_name */ + { 303, 0 }, /* (250) stream_options ::= */ + { 303, -3 }, /* (251) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 303, -3 }, /* (252) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 303, -3 }, /* (253) stream_options ::= stream_options WATERMARK duration_literal */ { 238, -3 }, /* (254) cmd ::= KILL CONNECTION NK_INTEGER */ { 238, -3 }, /* (255) cmd ::= KILL QUERY NK_INTEGER */ { 238, -3 }, /* (256) cmd ::= KILL TRANSACTION NK_INTEGER */ { 238, -4 }, /* (257) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { 238, -4 }, /* (258) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ { 238, -3 }, /* (259) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 304, -2 }, /* (260) dnode_list ::= DNODE NK_INTEGER */ - { 304, -3 }, /* (261) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 305, -2 }, /* (260) dnode_list ::= DNODE NK_INTEGER */ + { 305, -3 }, /* (261) dnode_list ::= dnode_list DNODE NK_INTEGER */ { 238, -3 }, /* (262) cmd ::= SYNCDB db_name REPLICA */ { 238, -1 }, /* (263) cmd ::= query_expression */ { 241, -1 }, /* (264) literal ::= NK_INTEGER */ @@ -2833,32 +2831,32 @@ static const struct { { 241, -1 }, /* (269) literal ::= duration_literal */ { 241, -1 }, /* (270) literal ::= NULL */ { 241, -1 }, /* (271) literal ::= NK_QUESTION */ - { 290, -1 }, /* (272) duration_literal ::= NK_VARIABLE */ - { 305, -1 }, /* (273) signed ::= NK_INTEGER */ - { 305, -2 }, /* (274) signed ::= NK_PLUS NK_INTEGER */ - { 305, -2 }, /* (275) signed ::= NK_MINUS NK_INTEGER */ - { 305, -1 }, /* (276) signed ::= NK_FLOAT */ - { 305, -2 }, /* (277) signed ::= NK_PLUS NK_FLOAT */ - { 305, -2 }, /* (278) signed ::= NK_MINUS NK_FLOAT */ - { 306, -1 }, /* (279) signed_literal ::= signed */ - { 306, -1 }, /* (280) signed_literal ::= NK_STRING */ - { 306, -1 }, /* (281) signed_literal ::= NK_BOOL */ - { 306, -2 }, /* (282) signed_literal ::= TIMESTAMP NK_STRING */ - { 306, -1 }, /* (283) signed_literal ::= duration_literal */ - { 306, -1 }, /* (284) signed_literal ::= NULL */ - { 306, -1 }, /* (285) signed_literal ::= literal_func */ - { 273, -1 }, /* (286) literal_list ::= signed_literal */ - { 273, -3 }, /* (287) literal_list ::= literal_list NK_COMMA signed_literal */ + { 291, -1 }, /* (272) duration_literal ::= NK_VARIABLE */ + { 306, -1 }, /* (273) signed ::= NK_INTEGER */ + { 306, -2 }, /* (274) signed ::= NK_PLUS NK_INTEGER */ + { 306, -2 }, /* (275) signed ::= NK_MINUS NK_INTEGER */ + { 306, -1 }, /* (276) signed ::= NK_FLOAT */ + { 306, -2 }, /* (277) signed ::= NK_PLUS NK_FLOAT */ + { 306, -2 }, /* (278) signed ::= NK_MINUS NK_FLOAT */ + { 271, -1 }, /* (279) signed_literal ::= signed */ + { 271, -1 }, /* (280) signed_literal ::= NK_STRING */ + { 271, -1 }, /* (281) signed_literal ::= NK_BOOL */ + { 271, -2 }, /* (282) signed_literal ::= TIMESTAMP NK_STRING */ + { 271, -1 }, /* (283) signed_literal ::= duration_literal */ + { 271, -1 }, /* (284) signed_literal ::= NULL */ + { 271, -1 }, /* (285) signed_literal ::= literal_func */ + { 274, -1 }, /* (286) literal_list ::= signed_literal */ + { 274, -3 }, /* (287) literal_list ::= literal_list NK_COMMA signed_literal */ { 248, -1 }, /* (288) db_name ::= NK_ID */ - { 276, -1 }, /* (289) table_name ::= NK_ID */ + { 277, -1 }, /* (289) table_name ::= NK_ID */ { 269, -1 }, /* (290) column_name ::= NK_ID */ - { 286, -1 }, /* (291) function_name ::= NK_ID */ + { 287, -1 }, /* (291) function_name ::= NK_ID */ { 308, -1 }, /* (292) table_alias ::= NK_ID */ { 309, -1 }, /* (293) column_alias ::= NK_ID */ { 243, -1 }, /* (294) user_name ::= NK_ID */ - { 287, -1 }, /* (295) index_name ::= NK_ID */ - { 294, -1 }, /* (296) topic_name ::= NK_ID */ - { 301, -1 }, /* (297) stream_name ::= NK_ID */ + { 288, -1 }, /* (295) index_name ::= NK_ID */ + { 295, -1 }, /* (296) topic_name ::= NK_ID */ + { 302, -1 }, /* (297) stream_name ::= NK_ID */ { 310, -1 }, /* (298) expression ::= literal */ { 310, -1 }, /* (299) expression ::= pseudo_column */ { 310, -1 }, /* (300) expression ::= column_reference */ @@ -2873,8 +2871,8 @@ static const struct { { 310, -3 }, /* (309) expression ::= expression NK_SLASH expression */ { 310, -3 }, /* (310) expression ::= expression NK_REM expression */ { 310, -3 }, /* (311) expression ::= column_reference NK_ARROW NK_STRING */ - { 293, -1 }, /* (312) expression_list ::= expression */ - { 293, -3 }, /* (313) expression_list ::= expression_list NK_COMMA expression */ + { 294, -1 }, /* (312) expression_list ::= expression */ + { 294, -3 }, /* (313) expression_list ::= expression_list NK_COMMA expression */ { 312, -1 }, /* (314) column_reference ::= column_name */ { 312, -3 }, /* (315) column_reference ::= table_name NK_DOT column_name */ { 311, -1 }, /* (316) pseudo_column ::= ROWTS */ @@ -2970,8 +2968,8 @@ static const struct { { 341, -4 }, /* (406) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ { 341, -6 }, /* (407) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ { 341, -8 }, /* (408) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 291, 0 }, /* (409) sliding_opt ::= */ - { 291, -4 }, /* (410) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 292, 0 }, /* (409) sliding_opt ::= */ + { 292, -4 }, /* (410) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ { 346, 0 }, /* (411) fill_opt ::= */ { 346, -4 }, /* (412) fill_opt ::= FILL NK_LP fill_mode NK_RP */ { 346, -6 }, /* (413) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ @@ -2986,7 +2984,7 @@ static const struct { { 348, -3 }, /* (422) group_by_list ::= group_by_list NK_COMMA expression */ { 343, 0 }, /* (423) having_clause_opt ::= */ { 343, -2 }, /* (424) having_clause_opt ::= HAVING search_condition */ - { 296, -4 }, /* (425) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 297, -4 }, /* (425) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { 349, -1 }, /* (426) query_expression_body ::= query_primary */ { 349, -4 }, /* (427) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ { 349, -3 }, /* (428) query_expression_body ::= query_expression_body UNION query_expression_body */ @@ -3501,8 +3499,8 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy172 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy172, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy105, &yymsp[0].minor.yy105); } yymsp[-4].minor.yy172 = yylhsminor.yy172; break; - case 121: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ -{ yylhsminor.yy172 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy172, &yymsp[-2].minor.yy105, releaseRawExprNode(pCxt, yymsp[0].minor.yy172)); } + case 121: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ +{ yylhsminor.yy172 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy172, &yymsp[-2].minor.yy105, yymsp[0].minor.yy172); } yymsp[-5].minor.yy172 = yylhsminor.yy172; break; case 123: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 9f8d5b48029f3004736a6ed9b77d14108adcba8c..b68ef2c591e0497c6f32a9ce69c9e1f229b5f92f 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -239,6 +239,10 @@ TEST_F(ParserSelectTest, semanticError) { // TSDB_CODE_PAR_WRONG_VALUE_TYPE run("SELECT timestamp '2010a' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); + run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); + + run("SELECT CEIL(LAST(ts, c1)) FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); + // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index a87c00bea9459de4eea33c8c1e082564231e70ca..1a97d9ab1b898b9b4a5dae1d4bade0e9b6d87bb8 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -248,6 +248,7 @@ static SLogicSubplan* unionCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) pSubplan->id.groupId = pCxt->groupId; pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->pNode = pNode; + pNode->pParent = NULL; return pSubplan; } @@ -408,17 +409,30 @@ static const SSplitRule splitRuleSet[] = {{.pName = "SuperTableScan", .splitFunc static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); +static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) { + char* pStr = NULL; + nodesNodeToString(pSubplan, false, &pStr, NULL); + qDebugL("apply %s rule: %s", pRuleName, pStr); + taosMemoryFree(pStr); +} + static int32_t applySplitRule(SLogicSubplan* pSubplan) { SSplitContext cxt = {.queryId = pSubplan->id.queryId, .groupId = pSubplan->id.groupId + 1, .split = false}; + bool split = false; do { - cxt.split = false; + split = false; for (int32_t i = 0; i < splitRuleNum; ++i) { + cxt.split = false; int32_t code = splitRuleSet[i].splitFunc(&cxt, pSubplan); if (TSDB_CODE_SUCCESS != code) { return code; } + if (cxt.split) { + split = true; + dumpLogicSubplan(splitRuleSet[i].pName, pSubplan); + } } - } while (cxt.split); + } while (split); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/planSetOpTest.cpp b/source/libs/planner/test/planSetOpTest.cpp index 717384aae69fe26973216c996aef199954225e23..5568d3cfbb994b83c3531957a33fc7ae1ff20eb3 100644 --- a/source/libs/planner/test/planSetOpTest.cpp +++ b/source/libs/planner/test/planSetOpTest.cpp @@ -23,7 +23,12 @@ class PlanSetOpTest : public PlannerTestBase {}; TEST_F(PlanSetOpTest, unionAll) { useDb("root", "test"); + // sql 1: single UNION ALL operator run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20"); + // sql 2: multi UNION ALL operator + run("SELECT c1, c2 FROM t1 WHERE c1 > 10 " + "UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20 " + "UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 30"); } TEST_F(PlanSetOpTest, unionAllSubquery) { @@ -44,7 +49,12 @@ TEST_F(PlanSetOpTest, unionAllWithSubquery) { TEST_F(PlanSetOpTest, union) { useDb("root", "test"); + // single UNION operator run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION SELECT c1, c2 FROM t1 WHERE c1 > 20"); + // multi UNION operator + run("SELECT c1, c2 FROM t1 WHERE c1 > 10 " + "UNION SELECT c1, c2 FROM t1 WHERE c1 > 20 " + "UNION SELECT c1, c2 FROM t1 WHERE c1 > 30"); } TEST_F(PlanSetOpTest, unionContainJoin) { @@ -62,3 +72,12 @@ TEST_F(PlanSetOpTest, unionSubquery) { run("SELECT * FROM (SELECT c1, c2 FROM t1 UNION SELECT c1, c2 FROM t1)"); } + +TEST_F(PlanSetOpTest, bug001) { + useDb("root", "test"); + + run("SELECT c2 FROM t1 WHERE c1 IS NOT NULL GROUP BY c2 " + "UNION " + "SELECT 'abcdefghijklmnopqrstuvwxyz' FROM t1 " + "WHERE 'abcdefghijklmnopqrstuvwxyz' IS NOT NULL GROUP BY 'abcdefghijklmnopqrstuvwxyz'"); +} diff --git a/source/libs/planner/test/planSubqueryTest.cpp b/source/libs/planner/test/planSubqueryTest.cpp index 6a7cb91bb931fc5d0d22e7cefd5c2156fcf4c74a..2d559c6f3b5322e4bd27bd571fc5e6829ccf262c 100644 --- a/source/libs/planner/test/planSubqueryTest.cpp +++ b/source/libs/planner/test/planSubqueryTest.cpp @@ -23,9 +23,7 @@ class PlanSubqeuryTest : public PlannerTestBase {}; TEST_F(PlanSubqeuryTest, basic) { useDb("root", "test"); - if (0 == g_skipSql) { - run("SELECT * FROM (SELECT * FROM t1)"); - } + run("SELECT * FROM (SELECT * FROM t1)"); run("SELECT LAST(c1) FROM (SELECT * FROM t1)"); } diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 36f66ddff689ed7f29acc571a34d2a3a7c1e6163..42c8558239b4389e0a0c469fdaa5842b53dc2398 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -25,7 +25,7 @@ class PlannerEnv : public testing::Environment { virtual void SetUp() { initMetaDataEnv(); generateMetaData(); - initLog("/tmp/td"); + initLog(TD_TMP_DIR_PATH "td"); } virtual void TearDown() { destroyMetaDataEnv(); } @@ -35,18 +35,19 @@ class PlannerEnv : public testing::Environment { private: void initLog(const char* path) { - dDebugFlag = 143; - vDebugFlag = 0; - mDebugFlag = 143; - cDebugFlag = 0; - jniDebugFlag = 0; - tmrDebugFlag = 135; - uDebugFlag = 135; - rpcDebugFlag = 143; - qDebugFlag = 143; - wDebugFlag = 0; - sDebugFlag = 0; - tsdbDebugFlag = 0; + int32_t logLevel = getLogLevel(); + dDebugFlag = logLevel; + vDebugFlag = logLevel; + mDebugFlag = logLevel; + cDebugFlag = logLevel; + jniDebugFlag = logLevel; + tmrDebugFlag = logLevel; + uDebugFlag = logLevel; + rpcDebugFlag = logLevel; + qDebugFlag = logLevel; + wDebugFlag = logLevel; + sDebugFlag = logLevel; + tsdbDebugFlag = logLevel; tsLogEmbedded = 1; tsAsyncLog = 0; @@ -60,17 +61,26 @@ class PlannerEnv : public testing::Environment { }; static void parseArg(int argc, char* argv[]) { - int opt = 0; - const char* optstring = ""; + int opt = 0; + const char* optstring = ""; + // clang-format off static struct option long_options[] = { - {"dump", optional_argument, NULL, 'd'}, {"skipSql", optional_argument, NULL, 's'}, {0, 0, 0, 0}}; + {"dump", optional_argument, NULL, 'd'}, + {"skipSql", required_argument, NULL, 's'}, + {"log", required_argument, NULL, 'l'}, + {0, 0, 0, 0} + }; + // clang-format on while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) { switch (opt) { case 'd': setDumpModule(optarg); break; case 's': - g_skipSql = 1; + setSkipSqlNum(optarg); + break; + case 'l': + setLogLevel(optarg); break; default: break; diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 94a28f46a82de78fd038f05682f0521796b716fe..6e184fec724ceffbaa186367956d1318c59d2fb9 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -48,6 +48,7 @@ enum DumpModule { DumpModule g_dumpModule = DUMP_MODULE_NOTHING; int32_t g_skipSql = 0; +int32_t g_logLevel = 131; void setDumpModule(const char* pModule) { if (NULL == pModule) { @@ -71,14 +72,26 @@ void setDumpModule(const char* pModule) { } } +void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(optarg); } + +void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); } + +int32_t getLogLevel() { return g_logLevel; } + class PlannerTestBaseImpl { public: void useDb(const string& acctId, const string& db) { caseEnv_.acctId_ = acctId; caseEnv_.db_ = db; + caseEnv_.nsql_ = g_skipSql; } void run(const string& sql) { + if (caseEnv_.nsql_ > 0) { + --(caseEnv_.nsql_); + return; + } + reset(); try { SQuery* pQuery = nullptr; @@ -109,6 +122,10 @@ class PlannerTestBaseImpl { } void prepare(const string& sql) { + if (caseEnv_.nsql_ > 0) { + return; + } + reset(); try { doParseSql(sql, &stmtEnv_.pQuery_, true); @@ -119,6 +136,10 @@ class PlannerTestBaseImpl { } void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) { + if (caseEnv_.nsql_ > 0) { + return; + } + try { doBindParams(stmtEnv_.pQuery_, pParams, colIdx); } catch (...) { @@ -128,6 +149,11 @@ class PlannerTestBaseImpl { } void exec() { + if (caseEnv_.nsql_ > 0) { + --(caseEnv_.nsql_); + return; + } + try { doParseBoundSql(stmtEnv_.pQuery_); @@ -157,8 +183,9 @@ class PlannerTestBaseImpl { private: struct caseEnv { - string acctId_; - string db_; + string acctId_; + string db_; + int32_t nsql_; }; struct stmtEnv { diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index 5a2050a45e91d262751333e2f8dd12b02690af2e..6b75573fb3f86a91446ef3ad492a51703af4f00f 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -37,8 +37,9 @@ class PlannerTestBase : public testing::Test { std::unique_ptr impl_; }; -extern int32_t g_skipSql; - -extern void setDumpModule(const char* pModule); +extern void setDumpModule(const char* pModule); +extern void setSkipSqlNum(const char* pNum); +extern void setLogLevel(const char* pLogLevel); +extern int32_t getLogLevel(); #endif // PLAN_TEST_UTIL_H diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index cb7ff08fa3b8c869a101ad202c274490a49091bd..5b358fb0080bef9f9df4a34a26a4816662154f59 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -725,7 +725,11 @@ void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) { qGetQueriedTableSchemaVersion(pTaskInfo, dbFName, tbName, &ctx->tbInfo.sversion, &ctx->tbInfo.tversion); - sprintf(ctx->tbInfo.tbFName, "%s.%s", dbFName, tbName); + if (dbFName[0] && tbName[0]) { + sprintf(ctx->tbInfo.tbFName, "%s.%s", dbFName, tbName); + } else { + ctx->tbInfo.tbFName[0] = 0; + } } int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { diff --git a/source/libs/stream/src/tstreamUpdate.c b/source/libs/stream/src/tstreamUpdate.c index 1197b6100a72c8438886addd3c2207f0b7961a6e..7921193fa2ca0500afc1467f0e05af267a21aa6c 100644 --- a/source/libs/stream/src/tstreamUpdate.c +++ b/source/libs/stream/src/tstreamUpdate.c @@ -17,21 +17,20 @@ #include "ttime.h" #define DEFAULT_FALSE_POSITIVE 0.01 -#define DEFAULT_BUCKET_SIZE 1024 -#define ROWS_PER_MILLISECOND 1 -#define MAX_NUM_SCALABLE_BF 120 -#define MIN_NUM_SCALABLE_BF 10 -#define DEFAULT_PREADD_BUCKET 1 -#define MAX_INTERVAL MILLISECOND_PER_MINUTE -#define MIN_INTERVAL (MILLISECOND_PER_SECOND * 10) +#define DEFAULT_BUCKET_SIZE 1024 +#define ROWS_PER_MILLISECOND 1 +#define MAX_NUM_SCALABLE_BF 120 +#define MIN_NUM_SCALABLE_BF 10 +#define DEFAULT_PREADD_BUCKET 1 +#define MAX_INTERVAL MILLISECOND_PER_MINUTE +#define MIN_INTERVAL (MILLISECOND_PER_SECOND * 10) static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { - if (pInfo->numSBFs < count ) { + if (pInfo->numSBFs < count) { count = pInfo->numSBFs; } for (uint64_t i = 0; i < count; ++i) { - SScalableBf *tsSBF = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, - DEFAULT_FALSE_POSITIVE); + SScalableBf *tsSBF = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, DEFAULT_FALSE_POSITIVE); taosArrayPush(pInfo->pTsSBFs, &tsSBF); } } @@ -76,7 +75,7 @@ static int64_t adjustWatermark(int64_t interval, int32_t watermark) { return watermark; } -SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark) { +SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark) { return updateInfoInit(pInterval->interval, pInterval->precision, watermark); } @@ -93,7 +92,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma uint64_t bfSize = (uint64_t)(pInfo->watermark / pInfo->interval); - pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(SScalableBf)); + pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void *)); if (pInfo->pTsSBFs == NULL) { updateInfoDestroy(pInfo); return NULL; @@ -108,14 +107,14 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } TSKEY dumy = 0; - for(uint64_t i=0; i < DEFAULT_BUCKET_SIZE; ++i) { + for (uint64_t i = 0; i < DEFAULT_BUCKET_SIZE; ++i) { taosArrayPush(pInfo->pTsBuckets, &dumy); } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; return pInfo; } -static SScalableBf* getSBf(SUpdateInfo *pInfo, TSKEY ts) { +static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { if (ts <= 0) { return NULL; } @@ -131,24 +130,23 @@ static SScalableBf* getSBf(SUpdateInfo *pInfo, TSKEY ts) { } SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index); if (res == NULL) { - res = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, - DEFAULT_FALSE_POSITIVE); + res = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, DEFAULT_FALSE_POSITIVE); taosArrayPush(pInfo->pTsSBFs, &res); } return res; } bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) { - int32_t res = TSDB_CODE_FAILED; - uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; - SScalableBf* pSBf = getSBf(pInfo, ts); + int32_t res = TSDB_CODE_FAILED; + uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; + SScalableBf *pSBf = getSBf(pInfo, ts); // pSBf may be a null pointer if (pSBf) { res = tScalableBfPut(pSBf, &ts, sizeof(TSKEY)); } TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index); - if (maxTs < ts ) { + if (maxTs < ts) { taosArraySet(pInfo->pTsBuckets, index, &ts); return false; } @@ -159,7 +157,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) { return false; } - //check from tsdb api + // check from tsdb api return true; } @@ -174,7 +172,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { SScalableBf *pSBF = taosArrayGetP(pInfo->pTsSBFs, i); tScalableBfDestroy(pSBF); } - + taosArrayDestroy(pInfo->pTsSBFs); taosMemoryFree(pInfo); -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 47c73745fd06697969d6440e61aab048522e7836..642e57243467819f2d7ef5e322a377c14d84f393 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -76,8 +76,8 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index)); if (pTmp != NULL) { memcpy(pStub, pTmp, sizeof(SRespStub)); - taosThreadMutexUnlock(&(pObj->mutex)); taosHashRemove(pObj->pRespHash, &index, sizeof(index)); + taosThreadMutexUnlock(&(pObj->mutex)); return 1; // get one object } taosThreadMutexUnlock(&(pObj->mutex)); @@ -90,4 +90,4 @@ void syncRespClean(SSyncRespMgr *pObj) { taosThreadMutexUnlock(&(pObj->mutex)); } -void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {} \ No newline at end of file +void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {} diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 58c3a83aff60bf1852534d73ebea8292264a15f1..d53c4a49ba068dd96fc8629efbbab3f0cedc36d9 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -16,7 +16,7 @@ class TfsTest : public ::testing::Test { protected: - static void SetUpTestSuite() { root = "/tmp/tfsTest"; } + static void SetUpTestSuite() { root = TD_TMP_DIR_PATH "tfsTest"; } static void TearDownTestSuite() {} public: @@ -299,15 +299,15 @@ TEST_F(TfsTest, 04_File) { TEST_F(TfsTest, 05_MultiDisk) { int32_t code = 0; - const char *root00 = "/tmp/tfsTest00"; - const char *root01 = "/tmp/tfsTest01"; - const char *root10 = "/tmp/tfsTest10"; - const char *root11 = "/tmp/tfsTest11"; - const char *root12 = "/tmp/tfsTest12"; - const char *root20 = "/tmp/tfsTest20"; - const char *root21 = "/tmp/tfsTest21"; - const char *root22 = "/tmp/tfsTest22"; - const char *root23 = "/tmp/tfsTest23"; + const char *root00 = TD_TMP_DIR_PATH "tfsTest00"; + const char *root01 = TD_TMP_DIR_PATH "tfsTest01"; + const char *root10 = TD_TMP_DIR_PATH "tfsTest10"; + const char *root11 = TD_TMP_DIR_PATH "tfsTest11"; + const char *root12 = TD_TMP_DIR_PATH "tfsTest12"; + const char *root20 = TD_TMP_DIR_PATH "tfsTest20"; + const char *root21 = TD_TMP_DIR_PATH "tfsTest21"; + const char *root22 = TD_TMP_DIR_PATH "tfsTest22"; + const char *root23 = TD_TMP_DIR_PATH "tfsTest23"; SDiskCfg dCfg[9] = {0}; tstrncpy(dCfg[0].dir, root01, TSDB_FILENAME_LEN); diff --git a/source/libs/transport/inc/rpcCache.h b/source/libs/transport/inc/rpcCache.h deleted file mode 100644 index 5148f5e23c928026c32f623e67fb7e576abd5604..0000000000000000000000000000000000000000 --- a/source/libs/transport/inc/rpcCache.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_RPC_CACHE_H -#define TDENGINE_RPC_CACHE_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, int64_t keepTimer); -void rpcCloseConnCache(void *handle); -void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType); -void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connType); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_RPC_CACHE_H diff --git a/source/libs/transport/inc/rpcLog.h b/source/libs/transport/inc/rpcLog.h deleted file mode 100644 index a5db866932c5228741de88b3a165def88950f238..0000000000000000000000000000000000000000 --- a/source/libs/transport/inc/rpcLog.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_RPC_LOG_H -#define TDENGINE_RPC_LOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tlog.h" - -#define tFatal(...) \ - { \ - if (rpcDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("RPC FATAL ", DEBUG_FATAL, rpcDebugFlag, __VA_ARGS__); \ - } \ - } -#define tError(...) \ - { \ - if (rpcDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("RPC ERROR ", DEBUG_ERROR, rpcDebugFlag, __VA_ARGS__); \ - } \ - } -#define tWarn(...) \ - { \ - if (rpcDebugFlag & DEBUG_WARN) { \ - taosPrintLog("RPC WARN ", DEBUG_WARN, rpcDebugFlag, __VA_ARGS__); \ - } \ - } -#define tInfo(...) \ - { \ - if (rpcDebugFlag & DEBUG_INFO) { \ - taosPrintLog("RPC ", DEBUG_INFO, rpcDebugFlag, __VA_ARGS__); \ - } \ - } -#define tDebug(...) \ - { \ - if (rpcDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("RPC ", DEBUG_DEBUG, rpcDebugFlag, __VA_ARGS__); \ - } \ - } -#define tTrace(...) \ - { \ - if (rpcDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); \ - } \ - } -#define tDump(x, y) \ - { \ - if (rpcDebugFlag & DEBUG_DUMP) { \ - taosDumpData((unsigned char *)x, y); \ - } \ - } - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_RPC_LOG_H diff --git a/source/libs/transport/inc/rpcTcp.h b/source/libs/transport/inc/rpcTcp.h deleted file mode 100644 index ad42307516462521055575d11bef38273b125fe1..0000000000000000000000000000000000000000 --- a/source/libs/transport/inc/rpcTcp.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _rpc_tcp_header_ -#define _rpc_tcp_header_ -#include - -#ifdef __cplusplus -extern "C" { -#endif - -void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); -void taosStopTcpServer(void *param); -void taosCleanUpTcpServer(void *param); - -void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle); -void taosStopTcpClient(void *chandle); -void taosCleanUpTcpClient(void *chandle); -void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port); - -void taosCloseTcpConnection(void *chandle); -int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/source/libs/transport/inc/rpcUdp.h b/source/libs/transport/inc/rpcUdp.h deleted file mode 100644 index 0c651d07ed4662305e107c09a7312f372aa98339..0000000000000000000000000000000000000000 --- a/source/libs/transport/inc/rpcUdp.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _rpc_udp_header_ -#define _rpc_udp_header_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "taosdef.h" - -void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int, void *fp, void *shandle); -void taosStopUdpConnection(void *handle); -void taosCleanUpUdpConnection(void *handle); -int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle); -void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port); - -void taosFreeMsgHdr(void *hdr); -int taosMsgHdrSize(void *hdr); -void taosSendMsgHdr(void *hdr, TdFilePtr pFile); -void taosInitMsgHdr(void **hdr, void *dest, int maxPkts); -void taosSetMsgHdrData(void *hdr, char *data, int dataLen); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index e4e79ccf476ce9a1253db536f623431810541ad5..e71e19edceedc32fa196fbdba37fb1d38ac62525 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -12,31 +12,20 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifdef USE_UV +#ifndef _TD_TRANSPORT_COMM_H +#define _TD_TRANSPORT_COMM_H #ifdef __cplusplus extern "C" { #endif #include -#include "lz4.h" #include "os.h" -#include "osSocket.h" -#include "rpcCache.h" -#include "rpcHead.h" -#include "rpcLog.h" #include "taoserror.h" -#include "tglobal.h" -#include "thash.h" #include "theap.h" -#include "tidpool.h" -#include "tmd5.h" -#include "tmempool.h" -#include "tmsg.h" +#include "transLog.h" #include "transportInt.h" -#include "tref.h" #include "trpc.h" -#include "ttimer.h" #include "tutil.h" typedef void* queue[2]; @@ -109,27 +98,6 @@ typedef void* queue[2]; #define TRANS_RETRY_INTERVAL 15 // ms retry interval #define TRANS_CONN_TIMEOUT 3 // connect timeout -typedef struct { - SRpcInfo* pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app - // struct SRpcConn* pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t* pCont; // content provided by app - int32_t contLen; // content length - // int32_t code; // error code - // int16_t numOfTry; // number of try for different servers - // int8_t oldInUse; // server EP inUse passed by app - // int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API - char* ip; - uint32_t port; - // SEpSet* pSet; // for synchronous API -} SRpcReqContext; - typedef SRpcMsg STransMsg; typedef SRpcCtx STransCtx; typedef SRpcCtxVal STransCtxVal; @@ -193,12 +161,7 @@ typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } Co #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) #define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) -#define RPC_MSG_OVERHEAD (sizeof(SRpcHead) + sizeof(SRpcDigest)) -#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) -#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) -#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) -#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) -#define rpcIsReq(type) (type & 1U) +#define rpcIsReq(type) (type & 1U) #define TRANS_RESERVE_SIZE (sizeof(STranConnCtx)) @@ -209,15 +172,14 @@ typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } Co #define transContLenFromMsg(msgLen) (msgLen - sizeof(STransMsgHead)); #define transIsReq(type) (type & 1U) -int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); -void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); -int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); -SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead); - -int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); -void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); -bool transCompressMsg(char* msg, int32_t len, int32_t* flen); -bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); +// int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); +// void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); +//// int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); +// +// int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); +// void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); +// bool transCompressMsg(char* msg, int32_t len, int32_t* flen); +// bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); void transFreeMsg(void* msg); @@ -365,4 +327,4 @@ void transThreadOnce(); } #endif -#endif +#endif // _TD_TRANSPORT_COMM_H diff --git a/source/libs/transport/inc/transLog.h b/source/libs/transport/inc/transLog.h new file mode 100644 index 0000000000000000000000000000000000000000..ebf231cfcff11aea08158264da5b71e77a429692 --- /dev/null +++ b/source/libs/transport/inc/transLog.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TRANSPORT_LOG_H +#define _TD_TRANSPORT_LOG_H + +#ifdef __cplusplus +extern "C" { +#endif + +// clang-format off +#include "tlog.h" + +#define tFatal(...) do {if (rpcDebugFlag & DEBUG_FATAL){ taosPrintLog("RPC FATAL ", DEBUG_FATAL, rpcDebugFlag, __VA_ARGS__); }} while (0) +#define tError(...)do { if (rpcDebugFlag & DEBUG_ERROR){ taosPrintLog("RPC ERROR ", DEBUG_ERROR, rpcDebugFlag, __VA_ARGS__); } } while(0) +#define tWarn(...) do { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", DEBUG_WARN, rpcDebugFlag, __VA_ARGS__); }} while(0) +#define tInfo(...) do { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC ", DEBUG_INFO, rpcDebugFlag, __VA_ARGS__); }} while(0) +#define tDebug(...) do {if (rpcDebugFlag & DEBUG_DEBUG){ taosPrintLog("RPC ", DEBUG_DEBUG, rpcDebugFlag, __VA_ARGS__); }} while(0) +#define tTrace(...) do {if (rpcDebugFlag & DEBUG_TRACE){ taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); }} while(0) +#define tDump(x, y) do {if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); } } while(0) +// clang-format on +#ifdef __cplusplus +} +#endif + +#endif // __TRANS_LOG_H diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index c8972067d824c04b088dd2762b8e19abc7af044d..8aeae1b5ade26a1a320dae37cbfe67f676f66eeb 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -21,18 +21,14 @@ #endif #include "lz4.h" #include "os.h" -#include "rpcCache.h" -#include "rpcHead.h" -#include "rpcLog.h" #include "taoserror.h" -#include "tglobal.h" #include "thash.h" -#include "tidpool.h" -#include "tmsg.h" #include "tref.h" +#include "tmsg.h" +#include "transLog.h" #include "trpc.h" -#include "ttimer.h" #include "tutil.h" +#include "tglobal.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c deleted file mode 100644 index e02de2d8b96f628d8eb07355690b968840378863..0000000000000000000000000000000000000000 --- a/source/libs/transport/src/rpcCache.c +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "rpcCache.h" -#include "os.h" -#include "rpcLog.h" -#include "taosdef.h" -#include "tglobal.h" -#include "tmempool.h" -#include "ttimer.h" -#include "tutil.h" - -typedef struct SConnHash { - char fqdn[TSDB_FQDN_LEN]; - uint16_t port; - char connType; - struct SConnHash *prev; - struct SConnHash *next; - void * data; - uint64_t time; -} SConnHash; - -typedef struct { - SConnHash ** connHashList; - mpool_h connHashMemPool; - int maxSessions; - int total; - int * count; - int64_t keepTimer; - TdThreadMutex mutex; - void (*cleanFp)(void *); - void * tmrCtrl; - void * pTimer; - int64_t *lockedBy; -} SConnCache; - -static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType); -static void rpcLockCache(int64_t *lockedBy); -static void rpcUnlockCache(int64_t *lockedBy); -static void rpcCleanConnCache(void *handle, void *tmrId); -static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time); - -void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, int64_t keepTimer) { - SConnHash **connHashList; - mpool_h connHashMemPool; - SConnCache *pCache; - - connHashMemPool = taosMemPoolInit(maxSessions, sizeof(SConnHash)); - if (connHashMemPool == 0) return NULL; - - connHashList = taosMemoryCalloc(sizeof(SConnHash *), maxSessions); - if (connHashList == 0) { - taosMemPoolCleanUp(connHashMemPool); - return NULL; - } - - pCache = taosMemoryMalloc(sizeof(SConnCache)); - if (pCache == NULL) { - taosMemPoolCleanUp(connHashMemPool); - taosMemoryFree(connHashList); - return NULL; - } - memset(pCache, 0, sizeof(SConnCache)); - - pCache->count = taosMemoryCalloc(sizeof(int), maxSessions); - pCache->total = 0; - pCache->keepTimer = keepTimer; - pCache->maxSessions = maxSessions; - pCache->connHashMemPool = connHashMemPool; - pCache->connHashList = connHashList; - pCache->cleanFp = cleanFp; - pCache->tmrCtrl = tmrCtrl; - pCache->lockedBy = taosMemoryCalloc(sizeof(int64_t), maxSessions); - taosTmrReset(rpcCleanConnCache, (int32_t)(pCache->keepTimer * 2), pCache, pCache->tmrCtrl, &pCache->pTimer); - - taosThreadMutexInit(&pCache->mutex, NULL); - - return pCache; -} - -void rpcCloseConnCache(void *handle) { - SConnCache *pCache; - - pCache = (SConnCache *)handle; - if (pCache == NULL || pCache->maxSessions == 0) return; - - taosThreadMutexLock(&pCache->mutex); - - taosTmrStopA(&(pCache->pTimer)); - - if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool); - - taosMemoryFreeClear(pCache->connHashList); - taosMemoryFreeClear(pCache->count); - taosMemoryFreeClear(pCache->lockedBy); - - taosThreadMutexUnlock(&pCache->mutex); - - taosThreadMutexDestroy(&pCache->mutex); - - memset(pCache, 0, sizeof(SConnCache)); - taosMemoryFree(pCache); -} - -void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType) { - int hash; - SConnHash * pNode; - SConnCache *pCache; - - uint64_t time = taosGetTimestampMs(); - - pCache = (SConnCache *)handle; - assert(pCache); - assert(data); - - hash = rpcHashConn(pCache, fqdn, port, connType); - pNode = (SConnHash *)taosMemPoolMalloc(pCache->connHashMemPool); - tstrncpy(pNode->fqdn, fqdn, sizeof(pNode->fqdn)); - pNode->port = port; - pNode->connType = connType; - pNode->data = data; - pNode->prev = NULL; - pNode->time = time; - - rpcLockCache(pCache->lockedBy + hash); - - pNode->next = pCache->connHashList[hash]; - if (pCache->connHashList[hash] != NULL) (pCache->connHashList[hash])->prev = pNode; - pCache->connHashList[hash] = pNode; - - pCache->count[hash]++; - rpcRemoveExpiredNodes(pCache, pNode->next, hash, time); - - rpcUnlockCache(pCache->lockedBy + hash); - - pCache->total++; - // tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, - // pCache->count[hash]); - - return; -} - -void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connType) { - int hash; - SConnHash * pNode; - SConnCache *pCache; - void * pData = NULL; - - pCache = (SConnCache *)handle; - assert(pCache); - - uint64_t time = taosGetTimestampMs(); - - hash = rpcHashConn(pCache, fqdn, port, connType); - rpcLockCache(pCache->lockedBy + hash); - - pNode = pCache->connHashList[hash]; - while (pNode) { - if (time >= pCache->keepTimer + pNode->time) { - rpcRemoveExpiredNodes(pCache, pNode, hash, time); - pNode = NULL; - break; - } - - if (strcmp(pNode->fqdn, fqdn) == 0 && pNode->port == port && pNode->connType == connType) break; - - pNode = pNode->next; - } - - if (pNode) { - rpcRemoveExpiredNodes(pCache, pNode->next, hash, time); - - if (pNode->prev) { - pNode->prev->next = pNode->next; - } else { - pCache->connHashList[hash] = pNode->next; - } - - if (pNode->next) { - pNode->next->prev = pNode->prev; - } - - pData = pNode->data; - taosMemPoolFree(pCache->connHashMemPool, (char *)pNode); - pCache->total--; - pCache->count[hash]--; - } - - rpcUnlockCache(pCache->lockedBy + hash); - - if (pData) { - // tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, - // pCache->count[hash]); - } else { - // tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, - // pCache->count[hash]); - } - - return pData; -} - -static void rpcCleanConnCache(void *handle, void *tmrId) { - int hash; - SConnHash * pNode; - SConnCache *pCache; - - pCache = (SConnCache *)handle; - if (pCache == NULL || pCache->maxSessions == 0) return; - if (pCache->pTimer != tmrId) return; - - taosThreadMutexLock(&pCache->mutex); - uint64_t time = taosGetTimestampMs(); - - for (hash = 0; hash < pCache->maxSessions; ++hash) { - rpcLockCache(pCache->lockedBy + hash); - pNode = pCache->connHashList[hash]; - rpcRemoveExpiredNodes(pCache, pNode, hash, time); - rpcUnlockCache(pCache->lockedBy + hash); - } - - // tTrace("timer, total connections in cache:%d", pCache->total); - taosTmrReset(rpcCleanConnCache, (int32_t)(pCache->keepTimer * 2), pCache, pCache->tmrCtrl, &pCache->pTimer); - taosThreadMutexUnlock(&pCache->mutex); -} - -static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time) { - if (pNode == NULL || (time < pCache->keepTimer + pNode->time)) return; - - SConnHash *pPrev = pNode->prev, *pNext; - - while (pNode) { - (*pCache->cleanFp)(pNode->data); - pNext = pNode->next; - pCache->total--; - pCache->count[hash]--; - // tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, - // pNode->connType, hash, pNode, - // pCache->count[hash]); - taosMemPoolFree(pCache->connHashMemPool, (char *)pNode); - pNode = pNext; - } - - if (pPrev) - pPrev->next = NULL; - else - pCache->connHashList[hash] = NULL; -} - -static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) { - SConnCache *pCache = (SConnCache *)handle; - int hash = 0; - char * temp = fqdn; - - while (*temp) { - hash += *temp; - ++temp; - } - - hash += port; - hash += connType; - - hash = hash % pCache->maxSessions; - - return hash; -} - -static void rpcLockCache(int64_t *lockedBy) { - int64_t tid = taosGetSelfPthreadId(); - int i = 0; - while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { - if (++i % 100 == 0) { - sched_yield(); - } - } -} - -static void rpcUnlockCache(int64_t *lockedBy) { - int64_t tid = taosGetSelfPthreadId(); - if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { - assert(false); - } -} diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c deleted file mode 100644 index 0e425970da7a600a2ac85431de4d4d4621f80a2d..0000000000000000000000000000000000000000 --- a/source/libs/transport/src/rpcMain.c +++ /dev/null @@ -1,1682 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "lz4.h" -#include "transportInt.h" -#include "os.h" -#include "rpcCache.h" -#include "rpcHead.h" -#include "rpcLog.h" -#include "rpcTcp.h" -#include "rpcUdp.h" -#include "taoserror.h" -#include "tglobal.h" -#include "thash.h" -#include "tidpool.h" -#include "tmd5.h" -#include "tmempool.h" -#include "tmsg.h" -#include "tref.h" -#include "trpc.h" -#include "ttimer.h" -#include "tutil.h" - -static TdThreadOnce tsRpcInitOnce = PTHREAD_ONCE_INIT; - -int tsRpcMaxUdpSize = 15000; // bytes -int tsProgressTimer = 100; -// not configurable -int tsRpcMaxRetry; -int tsRpcHeadSize; -int tsRpcOverhead; - -SHashObj *tsFqdnHash; - -#ifndef USE_UV - -typedef struct { - int sessions; // number of sessions allowed - int numOfThreads; // number of threads to process incoming messages - int idleTime; // milliseconds; - uint16_t localPort; - int8_t connType; - int index; // for UDP server only, round robin for multiple threads - char label[TSDB_LABEL_LEN]; - - char user[TSDB_UNI_LEN]; // meter ID - char spi; // security parameter index - char encrypt; // encrypt algorithm - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key - - void (*cfp)(void *parent, SRpcMsg *, SEpSet *); - int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); - - int32_t refCount; - void * parent; - void * idPool; // handle to ID pool - void * tmrCtrl; // handle to timer - SHashObj * hash; // handle returned by hash utility - void * tcphandle; // returned handle from TCP initialization - void * udphandle; // returned handle from UDP initialization - void * pCache; // connection cache - TdThreadMutex mutex; - struct SRpcConn *connList; // connection list -} SRpcInfo; - -typedef struct { - SRpcInfo * pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void * ahandle; // handle provided by app - struct SRpcConn *pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t * pCont; // content provided by app - int32_t contLen; // content length - int32_t code; // error code - int16_t numOfTry; // number of try for different servers - int8_t oldInUse; // server EP inUse passed by app - int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg * pRsp; // for synchronous API - tsem_t * pSem; // for synchronous API - SEpSet * pSet; // for synchronous API - char msg[0]; // RpcHead starts from here -} SRpcReqContext; - -#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) -#define rpcHeadFromCont(cont) ((SRpcHead *)((char *)cont - sizeof(SRpcHead))) -#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) -#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) -#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) -#define rpcIsReq(type) (type & 1U) - -typedef struct SRpcConn { - char info[48]; // debug info: label + pConn + ahandle - int sid; // session ID - uint32_t ownId; // own link ID - uint32_t peerId; // peer link ID - char user[TSDB_UNI_LEN]; // user ID for the link - char spi; // security parameter index - char encrypt; // encryption, 0:1 - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key - char secured; // if set to 1, no authentication - uint16_t localPort; // for UDP only - uint32_t linkUid; // connection unique ID assigned by client - uint32_t peerIp; // peer IP - uint16_t peerPort; // peer port - char peerFqdn[TSDB_FQDN_LEN]; // peer FQDN or ip string - uint16_t tranId; // outgoing transcation ID, for build message - uint16_t outTranId; // outgoing transcation ID - uint16_t inTranId; // transcation ID for incoming msg - tmsg_t outType; // message type for outgoing request - tmsg_t inType; // message type for incoming request - void * chandle; // handle passed by TCP/UDP connection layer - void * ahandle; // handle provided by upper app layter - int retry; // number of retry for sending request - int tretry; // total retry - void * pTimer; // retry timer to monitor the response - void * pIdleTimer; // idle timer - char * pRspMsg; // response message including header - int rspMsgLen; // response messag length - char * pReqMsg; // request message including header - int reqMsgLen; // request message length - SRpcInfo * pRpc; // the associated SRpcInfo - int8_t connType; // connection type - int64_t lockedBy; // lock for connection - SRpcReqContext *pContext; // request context -} SRpcConn; - -static int tsRpcRefId = -1; -static int32_t tsRpcNum = 0; - -// static TdThreadOnce tsRpcInit = PTHREAD_ONCE_INIT; - -// server:0 client:1 tcp:2 udp:0 -#define RPC_CONN_UDPS 0 -#define RPC_CONN_UDPC 1 -#define RPC_CONN_TCPS 2 -#define RPC_CONN_TCPC 3 - -void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = { - taosInitUdpConnection, taosInitUdpConnection, taosInitTcpServer, taosInitTcpClient}; - -void (*taosCleanUpConn[])(void *thandle) = {taosCleanUpUdpConnection, taosCleanUpUdpConnection, taosCleanUpTcpServer, - taosCleanUpTcpClient}; - -void (*taosStopConn[])(void *thandle) = { - taosStopUdpConnection, - taosStopUdpConnection, - taosStopTcpServer, - taosStopTcpClient, -}; - -int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = { - taosSendUdpData, taosSendUdpData, taosSendTcpData, taosSendTcpData}; - -void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port) = { - taosOpenUdpConnection, - taosOpenUdpConnection, - NULL, - taosOpenTcpClientConnection, -}; - -void (*taosCloseConn[])(void *chandle) = {NULL, NULL, taosCloseTcpConnection, taosCloseTcpConnection}; - -static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType); -static void rpcCloseConn(void *thandle); -static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext); -static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc); -static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv); -static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv); - -static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext); -static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code); -static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code); -static void rpcSendMsgToPeer(SRpcConn *pConn, void *data, int dataLen); -static void rpcSendReqHead(SRpcConn *pConn); - -static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv); -static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext); -static void rpcProcessConnError(void *param, void *id); -static void rpcProcessRetryTimer(void *, void *); -static void rpcProcessIdleTimer(void *param, void *tmrId); -static void rpcProcessProgressTimer(void *param, void *tmrId); - -static void rpcFreeMsg(void *msg); -static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen); -static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead); -static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen); -static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen); -static void rpcLockConn(SRpcConn *pConn); -static void rpcUnlockConn(SRpcConn *pConn); -static void rpcAddRef(SRpcInfo *pRpc); -static void rpcDecRef(SRpcInfo *pRpc); - -static void rpcFree(void *p) { - tTrace("free mem: %p", p); - taosMemoryFree(p); -} - -static void rpcInitImp(void) { - tsProgressTimer = tsRpcTimer / 2; - tsRpcMaxRetry = tsRpcMaxTime * 1000 / tsProgressTimer; - tsRpcHeadSize = RPC_MSG_OVERHEAD; - tsRpcOverhead = sizeof(SRpcReqContext); - - tsRpcRefId = taosOpenRef(200, rpcFree); - - tsFqdnHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); -} - -int32_t rpcInit() { - taosThreadOnce(&tsRpcInitOnce, rpcInitImp); - return 0; -} - -void rpcCleanup(void) { - taosCloseRef(tsRpcRefId); - taosHashClear(tsFqdnHash); - taosHashCleanup(tsFqdnHash); - tsFqdnHash = NULL; - tsRpcRefId = -1; -} - -void *rpcOpen(const SRpcInit *pInit) { - SRpcInfo *pRpc; - - // taosThreadOnce(&tsRpcInit, rpcInit); - - pRpc = (SRpcInfo *)taosMemoryCalloc(1, sizeof(SRpcInfo)); - if (pRpc == NULL) return NULL; - - if (pInit->label) tstrncpy(pRpc->label, pInit->label, tListLen(pInit->label)); - - pRpc->connType = pInit->connType; - if (pRpc->connType == TAOS_CONN_CLIENT) { - pRpc->numOfThreads = pInit->numOfThreads; - if (pRpc->numOfThreads >= 10) { - pRpc->numOfThreads = 10; - } - } else { - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - } - pRpc->idleTime = pInit->idleTime; - pRpc->localPort = pInit->localPort; - pRpc->afp = pInit->afp; - pRpc->sessions = pInit->sessions + 1; - if (pInit->user) tstrncpy(pRpc->user, pInit->user, sizeof(pRpc->user)); - if (pInit->secret) memcpy(pRpc->secret, pInit->secret, sizeof(pRpc->secret)); - if (pInit->ckey) tstrncpy(pRpc->ckey, pInit->ckey, sizeof(pRpc->ckey)); - pRpc->spi = pInit->spi; - pRpc->cfp = pInit->cfp; - pRpc->afp = pInit->afp; - pRpc->parent = pInit->parent; - pRpc->refCount = 1; - - atomic_add_fetch_32(&tsRpcNum, 1); - - size_t size = sizeof(SRpcConn) * pRpc->sessions; - pRpc->connList = (SRpcConn *)taosMemoryCalloc(1, size); - if (pRpc->connList == NULL) { - tError("%s failed to allocate memory for taos connections, size:%" PRId64, pRpc->label, (int64_t)size); - rpcClose(pRpc); - return NULL; - } - - pRpc->idPool = taosInitIdPool(pRpc->sessions - 1); - if (pRpc->idPool == NULL) { - tError("%s failed to init ID pool", pRpc->label); - rpcClose(pRpc); - return NULL; - } - - pRpc->tmrCtrl = taosTmrInit(pRpc->sessions * 2 + 1, 50, 10000, pRpc->label); - if (pRpc->tmrCtrl == NULL) { - tError("%s failed to init timers", pRpc->label); - rpcClose(pRpc); - return NULL; - } - - if (pRpc->connType == TAOS_CONN_SERVER) { - pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); - if (pRpc->hash == NULL) { - tError("%s failed to init string hash", pRpc->label); - rpcClose(pRpc); - return NULL; - } - } else { - pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime * 20); - if (pRpc->pCache == NULL) { - tError("%s failed to init connection cache", pRpc->label); - rpcClose(pRpc); - return NULL; - } - } - - taosThreadMutexInit(&pRpc->mutex, NULL); - - pRpc->tcphandle = (*taosInitConn[pRpc->connType | RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, - rpcProcessMsgFromPeer, pRpc); - pRpc->udphandle = - (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); - - if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) { - tError("%s failed to init network, port:%d", pRpc->label, pRpc->localPort); - rpcClose(pRpc); - return NULL; - } - - tDebug("%s rpc is opened, threads:%d sessions:%d", pRpc->label, pRpc->numOfThreads, pInit->sessions); - - return pRpc; -} - -void rpcClose(void *param) { - SRpcInfo *pRpc = (SRpcInfo *)param; - - // stop connection to outside first - (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); - (*taosStopConn[pRpc->connType])(pRpc->udphandle); - - // close all connections - for (int i = 0; i < pRpc->sessions; ++i) { - if (pRpc->connList && pRpc->connList[i].user[0]) { - rpcCloseConn((void *)(pRpc->connList + i)); - } - } - - // clean up - (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); - (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle); - - tDebug("%s rpc is closed", pRpc->label); - rpcDecRef(pRpc); -} - -void *rpcMallocCont(int contLen) { - int size = contLen + RPC_MSG_OVERHEAD; - - char *start = (char *)taosMemoryCalloc(1, (size_t)size); - if (start == NULL) { - tError("failed to malloc msg, size:%d", size); - return NULL; - } else { - tTrace("malloc mem:%p size:%d", start, size); - } - - return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); -} - -void rpcFreeCont(void *cont) { - if (cont) { - char *temp = ((char *)cont) - sizeof(SRpcHead) - sizeof(SRpcReqContext); - taosMemoryFree(temp); - tTrace("free mem: %p", temp); - } -} - -void *rpcReallocCont(void *ptr, int contLen) { - if (ptr == NULL) return rpcMallocCont(contLen); - - char *start = ((char *)ptr) - sizeof(SRpcReqContext) - sizeof(SRpcHead); - if (contLen == 0) { - taosMemoryFree(start); - return NULL; - } - - int size = contLen + RPC_MSG_OVERHEAD; - start = taosMemoryRealloc(start, size); - if (start == NULL) { - tError("failed to realloc cont, size:%d", size); - return NULL; - } - - return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); -} - -void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { - SRpcInfo * pRpc = (SRpcInfo *)shandle; - SRpcReqContext *pContext; - - int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); - pContext = (SRpcReqContext *)((char *)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); - pContext->ahandle = pMsg->ahandle; - pContext->pRpc = (SRpcInfo *)shandle; - pContext->epSet = *pEpSet; - pContext->contLen = contLen; - pContext->pCont = pMsg->pCont; - pContext->msgType = pMsg->msgType; - pContext->oldInUse = pEpSet->inUse; - - pContext->connType = RPC_CONN_UDPC; - if (contLen > tsRpcMaxUdpSize || tsRpcForceTcp) pContext->connType = RPC_CONN_TCPC; - - // connection type is application specific. - // for TDengine, all the query, show commands shall have TCP connection - tmsg_t type = pMsg->msgType; - if (type == TDMT_VND_QUERY || type == TDMT_MND_SHOW_RETRIEVE || type == TDMT_VND_FETCH || - type == TDMT_MND_VGROUP_LIST || type == TDMT_VND_TABLES_META || type == TDMT_VND_TABLE_META || - type == TDMT_MND_SHOW || type == TDMT_MND_STATUS || type == TDMT_VND_ALTER_TABLE) - pContext->connType = RPC_CONN_TCPC; - - pContext->rid = taosAddRef(tsRpcRefId, pContext); - if (pRid) *pRid = pContext->rid; - - rpcSendReqToServer(pRpc, pContext); -} - -void rpcSendResponse(const SRpcMsg *pRsp) { - ASSERT(pRsp->handle != NULL); - - int msgLen = 0; - SRpcConn *pConn = (SRpcConn *)pRsp->handle; - SRpcMsg rpcMsg = *pRsp; - SRpcMsg * pMsg = &rpcMsg; - SRpcInfo *pRpc = pConn->pRpc; - - if (pMsg->pCont == NULL) { - pMsg->pCont = rpcMallocCont(0); - pMsg->contLen = 0; - } - - SRpcHead *pHead = rpcHeadFromCont(pMsg->pCont); - char * msg = (char *)pHead; - - pMsg->contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); - msgLen = rpcMsgLenFromCont(pMsg->contLen); - - rpcLockConn(pConn); - - if (pConn->inType == 0 || pConn->user[0] == 0) { - tError("%s, connection is already released, rsp wont be sent", pConn->info); - rpcUnlockConn(pConn); - rpcFreeCont(pMsg->pCont); - rpcDecRef(pRpc); - return; - } - - // set msg header - pHead->version = 1; - pHead->msgType = pConn->inType + 1; - pHead->spi = pConn->spi; - pHead->encrypt = pConn->encrypt; - pHead->tranId = pConn->inTranId; - pHead->sourceId = pConn->ownId; - pHead->destId = pConn->peerId; - pHead->linkUid = pConn->linkUid; - pHead->port = htons(pConn->localPort); - pHead->code = htonl(pMsg->code); - pHead->ahandle = (uint64_t)pConn->ahandle; - - // set pConn parameters - pConn->inType = 0; - - // response message is released until new response is sent - rpcFreeMsg(pConn->pRspMsg); - pConn->pRspMsg = msg; - pConn->rspMsgLen = msgLen; - if (pMsg->code == TSDB_CODE_RPC_ACTION_IN_PROGRESS) pConn->inTranId--; - - // stop the progress timer - taosTmrStopA(&pConn->pTimer); - - // set the idle timer to monitor the activity - taosTmrReset(rpcProcessIdleTimer, pRpc->idleTime * 30, pConn, pRpc->tmrCtrl, &pConn->pIdleTimer); - rpcSendMsgToPeer(pConn, msg, msgLen); - - // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured - if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) pConn->secured = 1; // connection shall be secured - - if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); - pConn->pReqMsg = NULL; - pConn->reqMsgLen = 0; - - rpcUnlockConn(pConn); - rpcDecRef(pRpc); // decrease the referene count - - return; -} - -void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) { - SRpcMsg rpcMsg; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - - rpcMsg.contLen = sizeof(SEpSet); - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - if (rpcMsg.pCont == NULL) return; - - memcpy(rpcMsg.pCont, pEpSet, sizeof(SEpSet)); - - rpcMsg.code = TSDB_CODE_RPC_REDIRECT; - rpcMsg.handle = thandle; - - rpcSendResponse(&rpcMsg); - - return; -} - -int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { - SRpcConn *pConn = (SRpcConn *)thandle; - if (pConn->user[0] == 0) return -1; - - pInfo->clientIp = pConn->peerIp; - pInfo->clientPort = pConn->peerPort; - // pInfo->serverIp = pConn->destIp; - - tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user)); - return 0; -} - -void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - SRpcReqContext *pContext; - pContext = (SRpcReqContext *)((char *)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); - - memset(pRsp, 0, sizeof(SRpcMsg)); - - tsem_t sem; - tsem_init(&sem, 0, 0); - pContext->pSem = &sem; - pContext->pRsp = pRsp; - pContext->pSet = pEpSet; - - rpcSendRequest(shandle, pEpSet, pMsg, NULL); - - tsem_wait(&sem); - tsem_destroy(&sem); - - return; -} - -// this API is used by server app to keep an APP context in case connection is broken -int rpcReportProgress(void *handle, char *pCont, int contLen) { - SRpcConn *pConn = (SRpcConn *)handle; - int code = 0; - - rpcLockConn(pConn); - - if (pConn->user[0]) { - // pReqMsg and reqMsgLen is re-used to store the context from app server - pConn->pReqMsg = pCont; - pConn->reqMsgLen = contLen; - } else { - tDebug("%s, rpc connection is already released", pConn->info); - rpcFreeCont(pCont); - code = -1; - } - - rpcUnlockConn(pConn); - return code; -} - -void rpcCancelRequest(int64_t rid) { - SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid); - if (pContext == NULL) return; - - rpcCloseConn(pContext->pConn); - - taosReleaseRef(tsRpcRefId, rid); -} - -static void rpcFreeMsg(void *msg) { - if (msg) { - char *temp = (char *)msg - sizeof(SRpcReqContext); - taosMemoryFree(temp); - tTrace("free mem: %p", temp); - } -} - -static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) { - SRpcConn *pConn; - - uint32_t peerIp = 0; - uint32_t *pPeerIp = taosHashGet(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1); - if (pPeerIp != NULL) { - peerIp = *pPeerIp; - } else { - peerIp = taosGetIpv4FromFqdn(peerFqdn); - if (peerIp != 0xFFFFFFFF) { - taosHashPut(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1, &peerIp, sizeof(peerIp)); - } - } - - if (peerIp == 0xFFFFFFFF) { - tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); - terrno = TSDB_CODE_RPC_FQDN_ERROR; - return NULL; - } - - pConn = rpcAllocateClientConn(pRpc); - - if (pConn) { - tstrncpy(pConn->peerFqdn, peerFqdn, sizeof(pConn->peerFqdn)); - pConn->peerIp = peerIp; - pConn->peerPort = peerPort; - tstrncpy(pConn->user, pRpc->user, sizeof(pConn->user)); - pConn->connType = connType; - - if (taosOpenConn[connType]) { - void *shandle = (connType & RPC_CONN_TCP) ? pRpc->tcphandle : pRpc->udphandle; - pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIp, pConn->peerPort); - if (pConn->chandle == NULL) { - tError("failed to connect to:%s:%d", taosIpStr(pConn->peerIp), pConn->peerPort); - - terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; - rpcCloseConn(pConn); - pConn = NULL; - } - } - } - - return pConn; -} - -static void rpcReleaseConn(SRpcConn *pConn) { - SRpcInfo *pRpc = pConn->pRpc; - if (pConn->user[0] == 0) return; - - pConn->user[0] = 0; - if (taosCloseConn[pConn->connType]) (*taosCloseConn[pConn->connType])(pConn->chandle); - - taosTmrStopA(&pConn->pTimer); - taosTmrStopA(&pConn->pIdleTimer); - - if (pRpc->connType == TAOS_CONN_SERVER) { - char hashstr[40] = {0}; - size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, - pConn->connType); - taosHashRemove(pRpc->hash, hashstr, size); - rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg - pConn->pRspMsg = NULL; - - // if server has ever reported progress, free content - if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); // do not use rpcFreeMsg - } else { - // if there is an outgoing message, free it - if (pConn->outType && pConn->pReqMsg) { - SRpcReqContext *pContext = pConn->pContext; - if (pContext) { - if (pContext->pRsp) { - // for synchronous API, post semaphore to unblock app - pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; - pContext->pRsp->pCont = NULL; - pContext->pRsp->contLen = 0; - tsem_post(pContext->pSem); - } - pContext->pConn = NULL; - taosRemoveRef(tsRpcRefId, pContext->rid); - } else { - assert(0); - } - } - } - - // memset could not be used, since lockeBy can not be reset - pConn->inType = 0; - pConn->outType = 0; - pConn->inTranId = 0; - pConn->outTranId = 0; - pConn->secured = 0; - pConn->peerId = 0; - pConn->peerIp = 0; - pConn->peerPort = 0; - pConn->pReqMsg = NULL; - pConn->reqMsgLen = 0; - pConn->pContext = NULL; - pConn->chandle = NULL; - - taosFreeId(pRpc->idPool, pConn->sid); - tDebug("%s, rpc connection is released", pConn->info); -} - -static void rpcCloseConn(void *thandle) { - SRpcConn *pConn = (SRpcConn *)thandle; - if (pConn == NULL) return; - - rpcLockConn(pConn); - - if (pConn->user[0]) rpcReleaseConn(pConn); - - rpcUnlockConn(pConn); -} - -static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { - SRpcConn *pConn = NULL; - - int sid = taosAllocateId(pRpc->idPool); - if (sid <= 0) { - tError("%s maximum number of sessions:%d is reached", pRpc->label, pRpc->sessions); - terrno = TSDB_CODE_RPC_MAX_SESSIONS; - } else { - pConn = pRpc->connList + sid; - - pConn->pRpc = pRpc; - pConn->sid = sid; - pConn->tranId = (uint16_t)(taosRand() & 0xFFFF); - pConn->ownId = htonl(pConn->sid); - pConn->linkUid = (uint32_t)((int64_t)pConn + taosGetPId() + (int64_t)pConn->tranId); - pConn->spi = pRpc->spi; - pConn->encrypt = pRpc->encrypt; - if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_PASSWORD_LEN); - tDebug("%s %p client connection is allocated, uid:0x%x", pRpc->label, pConn, pConn->linkUid); - } - - return pConn; -} - -static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { - SRpcConn *pConn = NULL; - char hashstr[40] = {0}; - SRpcHead *pHead = (SRpcHead *)pRecv->msg; - - size_t size = - snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); - - // check if it is already allocated - SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); - if (ppConn) pConn = *ppConn; - if (pConn) { - pConn->secured = 0; - return pConn; - } - - // if code is not 0, it means it is simple reqhead, just ignore - if (pHead->code != 0) { - terrno = TSDB_CODE_RPC_ALREADY_PROCESSED; - return NULL; - } - - int sid = taosAllocateId(pRpc->idPool); - if (sid <= 0) { - tError("%s maximum number of sessions:%d is reached", pRpc->label, pRpc->sessions); - terrno = TSDB_CODE_RPC_MAX_SESSIONS; - } else { - pConn = pRpc->connList + sid; - memcpy(pConn->user, pHead->user, tListLen(pConn->user)); - pConn->pRpc = pRpc; - pConn->sid = sid; - pConn->tranId = (uint16_t)(taosRand() & 0xFFFF); - pConn->ownId = htonl(pConn->sid); - pConn->linkUid = pHead->linkUid; - if (pRpc->afp) { - if (pConn->user[0] == 0) { - terrno = TSDB_CODE_RPC_AUTH_REQUIRED; - } else { - terrno = (*pRpc->afp)(pRpc->parent, pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey); - } - - if (terrno != 0) { - taosFreeId(pRpc->idPool, sid); // sid shall be released - pConn = NULL; - } - } - } - - if (pConn) { - if (pRecv->connType == RPC_CONN_UDPS && pRpc->numOfThreads > 1) { - // UDP server, assign to new connection - pRpc->index = (pRpc->index + 1) % pRpc->numOfThreads; - pConn->localPort = (pRpc->localPort + pRpc->index); - } - - taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); - tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s spi:%d", pRpc->label, pConn, pConn->linkUid, - sid, hashstr, pConn->spi); - } - - return pConn; -} - -static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { - SRpcConn *pConn = NULL; - SRpcHead *pHead = (SRpcHead *)pRecv->msg; - - if (sid) { - pConn = pRpc->connList + sid; - if (pConn->user[0] == 0) pConn = NULL; - } - - if (pConn == NULL) { - if (pRpc->connType == TAOS_CONN_SERVER) { - pConn = rpcAllocateServerConn(pRpc, pRecv); - } else { - terrno = TSDB_CODE_RPC_UNEXPECTED_RESPONSE; - } - } - - if (pConn) { - if (pConn->linkUid != pHead->linkUid) { - terrno = TSDB_CODE_RPC_MISMATCHED_LINK_ID; - tDebug("%s %p %p, linkUid:0x%x is not matched with received:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, - pConn->linkUid, pHead->linkUid); - pConn = NULL; - } - } - - return pConn; -} - -static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { - SRpcConn *pConn; - SRpcInfo *pRpc = pContext->pRpc; - SEpSet * pEpSet = &pContext->epSet; - - pConn = rpcGetConnFromCache(pRpc->pCache, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, - pContext->connType); - if (pConn == NULL || pConn->user[0] == 0) { - pConn = rpcOpenConn(pRpc, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, pContext->connType); - } - - if (pConn) { - pConn->tretry = 0; - pConn->ahandle = pContext->ahandle; - snprintf(pConn->info, sizeof(pConn->info), "%s %p %p", pRpc->label, pConn, pConn->ahandle); - pConn->tretry = 0; - } else { - tError("%s %p, failed to set up connection(%s)", pRpc->label, pContext->ahandle, tstrerror(terrno)); - } - - return pConn; -} - -static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) { - if (pConn->peerId == 0) { - pConn->peerId = pHead->sourceId; - } else { - if (pConn->peerId != pHead->sourceId) { - tDebug("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info, pConn->peerId, pHead->sourceId); - return TSDB_CODE_RPC_INVALID_VALUE; - } - } - - if (pConn->inTranId == pHead->tranId) { - if (pConn->inType == pHead->msgType) { - if (pHead->code == 0) { - tDebug("%s, %s is retransmitted", pConn->info, TMSG_INFO(pHead->msgType)); - rpcSendQuickRsp(pConn, TSDB_CODE_RPC_ACTION_IN_PROGRESS); - } else { - // do nothing, it is heart beat from client - } - } else if (pConn->inType == 0) { - tDebug("%s, %s is already processed, tranId:%d", pConn->info, TMSG_INFO(pHead->msgType), pConn->inTranId); - rpcSendMsgToPeer(pConn, pConn->pRspMsg, pConn->rspMsgLen); // resend the response - } else { - tDebug("%s, mismatched message %s and tranId", pConn->info, TMSG_INFO(pHead->msgType)); - } - - // do not reply any message - return TSDB_CODE_RPC_ALREADY_PROCESSED; - } - - if (pConn->inType != 0) { - tDebug("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info, pConn->inTranId, pHead->tranId); - return TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED; - } - - if (rpcContLenFromMsg(pHead->msgLen) <= 0) { - tDebug("%s, message body is empty, ignore", pConn->info); - return TSDB_CODE_RPC_APP_ERROR; - } - - pConn->inTranId = pHead->tranId; - pConn->inType = pHead->msgType; - - // start the progress timer to monitor the response from server app - if (pConn->connType != RPC_CONN_TCPS) - pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pConn->pRpc->tmrCtrl); - - return 0; -} - -static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { - SRpcInfo *pRpc = pConn->pRpc; - pConn->peerId = pHead->sourceId; - - if (pConn->outType == 0 || pConn->pContext == NULL) { - return TSDB_CODE_RPC_UNEXPECTED_RESPONSE; - } - - if (pHead->tranId != pConn->outTranId) { - return TSDB_CODE_RPC_INVALID_TRAN_ID; - } - - if (pHead->msgType != pConn->outType + 1) { - return TSDB_CODE_RPC_INVALID_RESPONSE_TYPE; - } - - taosTmrStopA(&pConn->pTimer); - pConn->retry = 0; - - if (pHead->code == TSDB_CODE_RPC_AUTH_REQUIRED && pRpc->spi) { - tDebug("%s, authentication shall be restarted", pConn->info); - pConn->secured = 0; - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); - if (pConn->connType != RPC_CONN_TCPC) - pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); - return TSDB_CODE_RPC_ALREADY_PROCESSED; - } - - if (pHead->code == TSDB_CODE_RPC_MISMATCHED_LINK_ID) { - tDebug("%s, mismatched linkUid, link shall be restarted", pConn->info); - pConn->secured = 0; - ((SRpcHead *)pConn->pReqMsg)->destId = 0; - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); - if (pConn->connType != RPC_CONN_TCPC) - pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); - return TSDB_CODE_RPC_ALREADY_PROCESSED; - } - - if (pHead->code == TSDB_CODE_RPC_ACTION_IN_PROGRESS) { - if (pConn->tretry <= tsRpcMaxRetry) { - tDebug("%s, peer is still processing the transaction, retry:%d", pConn->info, pConn->tretry); - pConn->tretry++; - rpcSendReqHead(pConn); - if (pConn->connType != RPC_CONN_TCPC) - pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); - return TSDB_CODE_RPC_ALREADY_PROCESSED; - } else { - // peer still in processing, give up - tDebug("%s, server processing takes too long time, give up", pConn->info); - pHead->code = TSDB_CODE_RPC_TOO_SLOW; - } - } - - pConn->outType = 0; - pConn->pReqMsg = NULL; - pConn->reqMsgLen = 0; - SRpcReqContext *pContext = pConn->pContext; - - if (pHead->code == TSDB_CODE_RPC_REDIRECT) { - if (rpcContLenFromMsg(pHead->msgLen) < sizeof(SEpSet)) { - // if EpSet is not included in the msg, treat it as NOT_READY - pHead->code = TSDB_CODE_RPC_NOT_READY; - } else { - pContext->redirect++; - if (pContext->redirect > TSDB_MAX_REPLICA) { - pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - tWarn("%s, too many redirects, quit", pConn->info); - } - } - } - - return TSDB_CODE_SUCCESS; -} - -static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqContext **ppContext) { - int32_t sid; - SRpcConn *pConn = NULL; - - SRpcHead *pHead = (SRpcHead *)pRecv->msg; - - sid = htonl(pHead->destId); - *ppContext = NULL; - - if (TMSG_INDEX(pHead->msgType) >= TDMT_MAX || TMSG_INDEX(pHead->msgType) <= 0) { - tDebug("%s sid:%d, invalid message type:%d", pRpc->label, sid, pHead->msgType); - terrno = TSDB_CODE_RPC_INVALID_MSG_TYPE; - return NULL; - } - - if (sid < 0 || sid >= pRpc->sessions) { - tDebug("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid, pRpc->sessions, - TMSG_INFO(pHead->msgType)); - terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; - return NULL; - } - - if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) { - tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, - TMSG_INFO(pHead->msgType)); - terrno = TSDB_CODE_RPC_INVALID_VERSION; - return NULL; - } - - pConn = rpcGetConnObj(pRpc, sid, pRecv); - if (pConn == NULL) { - tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); - return NULL; - } - - rpcLockConn(pConn); - - if (rpcIsReq(pHead->msgType)) { - pConn->ahandle = (void *)pHead->ahandle; - snprintf(pConn->info, sizeof(pConn->info), "%s %p %p", pRpc->label, pConn, pConn->ahandle); - } - - sid = pConn->sid; - if (pConn->chandle == NULL) pConn->chandle = pRecv->chandle; - pConn->peerIp = pRecv->ip; - pConn->peerPort = pRecv->port; - if (pHead->port) pConn->peerPort = htons(pHead->port); - - terrno = rpcCheckAuthentication(pConn, (char *)pHead, pRecv->msgLen); - - // code can be transformed only after authentication - pHead->code = htonl(pHead->code); - - if (terrno == 0) { - if (pHead->encrypt) { - // decrypt here - } - - if (rpcIsReq(pHead->msgType)) { - pConn->connType = pRecv->connType; - terrno = rpcProcessReqHead(pConn, pHead); - - // stop idle timer - taosTmrStopA(&pConn->pIdleTimer); - - // client shall send the request within tsRpcTime again for UDP, double it - if (pConn->connType != RPC_CONN_TCPS) - pConn->pIdleTimer = taosTmrStart(rpcProcessIdleTimer, tsRpcTimer * 2, pConn, pRpc->tmrCtrl); - } else { - terrno = rpcProcessRspHead(pConn, pHead); - *ppContext = pConn->pContext; - } - } - - rpcUnlockConn(pConn); - - return pConn; -} - -static void doRpcReportBrokenLinkToServer(void *param, void *id) { - SRpcMsg * pRpcMsg = (SRpcMsg *)(param); - SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); - SRpcInfo *pRpc = pConn->pRpc; - (*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL); - taosMemoryFree(pRpcMsg); -} -static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { - SRpcInfo *pRpc = pConn->pRpc; - if (pConn->pReqMsg == NULL) return; - - // if there are pending request, notify the app - rpcAddRef(pRpc); - tDebug("%s, notify the server app, connection is gone", pConn->info); - - SRpcMsg *rpcMsg = taosMemoryMalloc(sizeof(SRpcMsg)); - rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server - rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length - rpcMsg->ahandle = pConn->ahandle; - rpcMsg->handle = pConn; - rpcMsg->msgType = pConn->inType; - rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - pConn->pReqMsg = NULL; - pConn->reqMsgLen = 0; - if (pRpc->cfp) { - taosTmrStart(doRpcReportBrokenLinkToServer, 0, rpcMsg, pRpc->tmrCtrl); - } else { - taosMemoryFree(rpcMsg); - } -} - -static void rpcProcessBrokenLink(SRpcConn *pConn) { - if (pConn == NULL) return; - SRpcInfo *pRpc = pConn->pRpc; - tDebug("%s, link is broken", pConn->info); - - rpcLockConn(pConn); - - if (pConn->outType) { - SRpcReqContext *pContext = pConn->pContext; - pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - pContext->pConn = NULL; - pConn->pReqMsg = NULL; - taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); - } - - if (pConn->inType) rpcReportBrokenLinkToServer(pConn); - - rpcReleaseConn(pConn); - rpcUnlockConn(pConn); -} - -static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { - SRpcHead *pHead = (SRpcHead *)pRecv->msg; - SRpcInfo *pRpc = (SRpcInfo *)pRecv->shandle; - SRpcConn *pConn = (SRpcConn *)pRecv->thandle; - - tDump(pRecv->msg, pRecv->msgLen); - - // underlying UDP layer does not know it is server or client - pRecv->connType = pRecv->connType | pRpc->connType; - - if (pRecv->msg == NULL) { - rpcProcessBrokenLink(pConn); - return NULL; - } - - terrno = 0; - SRpcReqContext *pContext; - pConn = rpcProcessMsgHead(pRpc, pRecv, &pContext); - - char ipstr[24] = {0}; - taosIpPort2String(pRecv->ip, pRecv->port, ipstr); - - if (TMSG_INDEX(pHead->msgType) >= 1 && TMSG_INDEX(pHead->msgType) < TDMT_MAX) { - tDebug("%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn, - (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), ipstr, terrno, pRecv->msgLen, pHead->sourceId, - pHead->destId, pHead->tranId, pHead->code); - } else { - tDebug("%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn, - (void *)pHead->ahandle, pHead->msgType, ipstr, terrno, pRecv->msgLen, pHead->sourceId, pHead->destId, - pHead->tranId, pHead->code); - } - - int32_t code = terrno; - if (code != TSDB_CODE_RPC_ALREADY_PROCESSED) { - if (code != 0) { // parsing error - if (rpcIsReq(pHead->msgType)) { - rpcSendErrorMsgToPeer(pRecv, code); - if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) { - rpcCloseConn(pConn); - } - if (TMSG_INDEX(pHead->msgType) + 1 > 1 && TMSG_INDEX(pHead->msgType) + 1 < TDMT_MAX) { - tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, - TMSG_INFO(pHead->msgType + 1), code); - } else { - tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, - TMSG_INFO(pHead->msgType), code); - } - } - } else { // msg is passed to app only parsing is ok - rpcProcessIncomingMsg(pConn, pHead, pContext); - } - } - - if (code) rpcFreeMsg(pRecv->msg); // parsing failed, msg shall be freed - return pConn; -} - -static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { - SRpcInfo *pRpc = pContext->pRpc; - - pContext->pConn = NULL; - if (pContext->pRsp) { - // for synchronous API - memcpy(pContext->pSet, &pContext->epSet, sizeof(SEpSet)); - memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg)); - tsem_post(pContext->pSem); - } else { - // for asynchronous API - SEpSet *pEpSet = NULL; - if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) pEpSet = &pContext->epSet; - - (*pRpc->cfp)(pRpc->parent, pMsg, pEpSet); - } - - // free the request message - taosRemoveRef(tsRpcRefId, pContext->rid); -} - -static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) { - SRpcInfo *pRpc = pConn->pRpc; - SRpcMsg rpcMsg; - - pHead = rpcDecompressRpcMsg(pHead); - rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen); - rpcMsg.pCont = pHead->content; - rpcMsg.msgType = pHead->msgType; - rpcMsg.code = pHead->code; - - if (rpcIsReq(pHead->msgType)) { - rpcMsg.ahandle = pConn->ahandle; - rpcMsg.handle = pConn; - rpcAddRef(pRpc); // add the refCount for requests - - // notify the server app - (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); - } else { - // it's a response - rpcMsg.handle = pContext; - rpcMsg.ahandle = pContext->ahandle; - pContext->pConn = NULL; - - // for UDP, port may be changed by server, the port in epSet shall be used for cache - if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { - rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.eps[pContext->epSet.inUse].port, - pConn->connType); - } else { - rpcCloseConn(pConn); - } - - if (pHead->code == TSDB_CODE_RPC_REDIRECT) { - pContext->numOfTry = 0; - SEpSet *pEpSet = (SEpSet *)pHead->content; - if (pEpSet->numOfEps > 0) { - memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); - tDebug("%s, redirect is received, numOfEps:%d inUse:%d", pConn->info, pContext->epSet.numOfEps, - pContext->epSet.inUse); - for (int i = 0; i < pContext->epSet.numOfEps; ++i) { - pContext->epSet.eps[i].port = htons(pContext->epSet.eps[i].port); - tDebug("%s, redirect is received, index:%d ep:%s:%u", pConn->info, i, pContext->epSet.eps[i].fqdn, - pContext->epSet.eps[i].port); - } - } - rpcSendReqToServer(pRpc, pContext); - rpcFreeCont(rpcMsg.pCont); - } else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY || - pHead->code == TSDB_CODE_NODE_OFFLINE) { - pContext->code = pHead->code; - rpcProcessConnError(pContext, NULL); - rpcFreeCont(rpcMsg.pCont); - } else { - rpcNotifyClient(pContext, &rpcMsg); - } - } -} - -static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) { - char msg[RPC_MSG_OVERHEAD]; - SRpcHead *pHead; - - // set msg header - memset(msg, 0, sizeof(SRpcHead)); - pHead = (SRpcHead *)msg; - pHead->version = 1; - pHead->msgType = pConn->inType + 1; - pHead->spi = pConn->spi; - pHead->encrypt = 0; - pHead->tranId = pConn->inTranId; - pHead->sourceId = pConn->ownId; - pHead->destId = pConn->peerId; - pHead->linkUid = pConn->linkUid; - pHead->ahandle = (uint64_t)pConn->ahandle; - memcpy(pHead->user, pConn->user, tListLen(pHead->user)); - pHead->code = htonl(code); - - rpcSendMsgToPeer(pConn, msg, sizeof(SRpcHead)); - pConn->secured = 1; // connection shall be secured -} - -static void rpcSendReqHead(SRpcConn *pConn) { - char msg[RPC_MSG_OVERHEAD]; - SRpcHead *pHead; - - // set msg header - memset(msg, 0, sizeof(SRpcHead)); - pHead = (SRpcHead *)msg; - pHead->version = 1; - pHead->msgType = pConn->outType; - pHead->msgVer = htonl(tsVersion >> 8); - pHead->spi = pConn->spi; - pHead->encrypt = 0; - pHead->tranId = pConn->outTranId; - pHead->sourceId = pConn->ownId; - pHead->destId = pConn->peerId; - pHead->linkUid = pConn->linkUid; - pHead->ahandle = (uint64_t)pConn->ahandle; - memcpy(pHead->user, pConn->user, tListLen(pHead->user)); - pHead->code = 1; - - rpcSendMsgToPeer(pConn, msg, sizeof(SRpcHead)); -} - -static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) { - SRpcHead *pRecvHead, *pReplyHead; - char msg[sizeof(SRpcHead) + sizeof(SRpcDigest) + sizeof(uint32_t)]; - uint32_t timeStamp; - int msgLen; - - pRecvHead = (SRpcHead *)pRecv->msg; - pReplyHead = (SRpcHead *)msg; - - memset(msg, 0, sizeof(SRpcHead)); - pReplyHead->version = pRecvHead->version; - pReplyHead->msgType = (tmsg_t)(pRecvHead->msgType + 1); - pReplyHead->spi = 0; - pReplyHead->encrypt = pRecvHead->encrypt; - pReplyHead->tranId = pRecvHead->tranId; - pReplyHead->sourceId = pRecvHead->destId; - pReplyHead->destId = pRecvHead->sourceId; - pReplyHead->linkUid = pRecvHead->linkUid; - pReplyHead->ahandle = pRecvHead->ahandle; - - pReplyHead->code = htonl(code); - msgLen = sizeof(SRpcHead); - - if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP) { - // include a time stamp if client's time is not synchronized well - uint8_t *pContent = pReplyHead->content; - timeStamp = htonl(taosGetTimestampSec()); - memcpy(pContent, &timeStamp, sizeof(timeStamp)); - msgLen += sizeof(timeStamp); - } - - pReplyHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - (*taosSendData[pRecv->connType])(pRecv->ip, pRecv->port, msg, msgLen, pRecv->chandle); - - return; -} - -static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { - SRpcHead *pHead = rpcHeadFromCont(pContext->pCont); - char * msg = (char *)pHead; - int msgLen = rpcMsgLenFromCont(pContext->contLen); - tmsg_t msgType = pContext->msgType; - - pContext->numOfTry++; - SRpcConn *pConn = rpcSetupConnToServer(pContext); - if (pConn == NULL) { - pContext->code = terrno; - taosTmrStart(rpcProcessConnError, 1, pContext, pRpc->tmrCtrl); - return; - } - - pContext->pConn = pConn; - pConn->ahandle = pContext->ahandle; - rpcLockConn(pConn); - - // set the message header - pHead->version = 1; - pHead->msgVer = htonl(tsVersion >> 8); - pHead->msgType = msgType; - pHead->encrypt = 0; - pConn->tranId++; - if (pConn->tranId == 0) pConn->tranId++; - pHead->tranId = pConn->tranId; - pHead->sourceId = pConn->ownId; - pHead->destId = pConn->peerId; - pHead->port = 0; - pHead->linkUid = pConn->linkUid; - pHead->ahandle = (uint64_t)pConn->ahandle; - memcpy(pHead->user, pConn->user, tListLen(pHead->user)); - - // set the connection parameters - pConn->outType = msgType; - pConn->outTranId = pHead->tranId; - pConn->pReqMsg = msg; - pConn->reqMsgLen = msgLen; - pConn->pContext = pContext; - - rpcSendMsgToPeer(pConn, msg, msgLen); - if (pConn->connType != RPC_CONN_TCPC) - taosTmrReset(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl, &pConn->pTimer); - - rpcUnlockConn(pConn); -} - -static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { - int writtenLen = 0; - SRpcHead *pHead = (SRpcHead *)msg; - - msgLen = rpcAddAuthPart(pConn, msg, msgLen); - - if (rpcIsReq(pHead->msgType)) { - tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), - pConn->peerFqdn, pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId); - } else { - if (pHead->code == 0) { - pConn->secured = 1; // for success response, set link as secured - } - - char ipport[40] = {0}; - taosIpPort2String(pConn->peerIp, pConn->peerPort, ipport); - tDebug("%s, %s is sent to %s, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), - ipport, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); - } - - // tTrace("connection type is: %d", pConn->connType); - writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle); - - if (writtenLen != msgLen) { - tError("%s, failed to send, msgLen:%d written:%d, reason:%s", pConn->info, msgLen, writtenLen, strerror(errno)); - } - - tDump(msg, msgLen); -} - -static void rpcProcessConnError(void *param, void *id) { - SRpcReqContext *pContext = (SRpcReqContext *)param; - SRpcInfo * pRpc = pContext->pRpc; - SRpcMsg rpcMsg = {0}; - - if (pRpc == NULL) { - return; - } - - tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle); - - if (pContext->numOfTry >= pContext->epSet.numOfEps || pContext->msgType == TDMT_VND_FETCH) { - rpcMsg.msgType = pContext->msgType + 1; - rpcMsg.ahandle = pContext->ahandle; - rpcMsg.code = pContext->code; - rpcMsg.pCont = NULL; - rpcMsg.contLen = 0; - - rpcNotifyClient(pContext, &rpcMsg); - } else { - // move to next IP - pContext->epSet.inUse++; - pContext->epSet.inUse = pContext->epSet.inUse % pContext->epSet.numOfEps; - rpcSendReqToServer(pRpc, pContext); - } -} - -static void rpcProcessRetryTimer(void *param, void *tmrId) { - SRpcConn *pConn = (SRpcConn *)param; - SRpcInfo *pRpc = pConn->pRpc; - - rpcLockConn(pConn); - - if (pConn->outType && pConn->user[0]) { - tDebug("%s, expected %s is not received", pConn->info, TMSG_INFO((int)pConn->outType + 1)); - pConn->pTimer = NULL; - pConn->retry++; - - if (pConn->retry < 4) { - tDebug("%s, re-send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, pConn->peerPort); - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); - pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); - } else { - // close the connection - tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, - pConn->peerPort); - if (pConn->pContext) { - pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - pConn->pContext->pConn = NULL; - pConn->pReqMsg = NULL; - taosTmrStart(rpcProcessConnError, 1, pConn->pContext, pRpc->tmrCtrl); - rpcReleaseConn(pConn); - } - } - } else { - tDebug("%s, retry timer not processed", pConn->info); - } - - rpcUnlockConn(pConn); -} - -static void rpcProcessIdleTimer(void *param, void *tmrId) { - SRpcConn *pConn = (SRpcConn *)param; - - rpcLockConn(pConn); - - if (pConn->user[0]) { - tDebug("%s, close the connection since no activity", pConn->info); - if (pConn->inType) rpcReportBrokenLinkToServer(pConn); - rpcReleaseConn(pConn); - } else { - tDebug("%s, idle timer:%p not processed", pConn->info, tmrId); - } - - rpcUnlockConn(pConn); -} - -static void rpcProcessProgressTimer(void *param, void *tmrId) { - SRpcConn *pConn = (SRpcConn *)param; - SRpcInfo *pRpc = pConn->pRpc; - - rpcLockConn(pConn); - - if (pConn->inType && pConn->user[0]) { - tDebug("%s, progress timer expired, send progress", pConn->info); - rpcSendQuickRsp(pConn, TSDB_CODE_RPC_ACTION_IN_PROGRESS); - pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pRpc->tmrCtrl); - } else { - tDebug("%s, progress timer:%p not processed", pConn->info, tmrId); - } - - rpcUnlockConn(pConn); -} - -static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) { - SRpcHead *pHead = rpcHeadFromCont(pCont); - int32_t finalLen = 0; - int overhead = sizeof(SRpcComp); - - if (!NEEDTO_COMPRESSS_MSG(contLen)) { - return contLen; - } - - char *buf = taosMemoryMalloc(contLen + overhead + 8); // 8 extra bytes - if (buf == NULL) { - tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); - return contLen; - } - - int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead); - tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead); - - /* - * only the compressed size is less than the value of contLen - overhead, the compression is applied - * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message - */ - if (compLen > 0 && compLen < contLen - overhead) { - SRpcComp *pComp = (SRpcComp *)pCont; - pComp->reserved = 0; - pComp->contLen = htonl(contLen); - memcpy(pCont + overhead, buf, compLen); - - pHead->comp = 1; - tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); - finalLen = compLen + overhead; - } else { - finalLen = contLen; - } - - taosMemoryFree(buf); - return finalLen; -} - -static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { - int overhead = sizeof(SRpcComp); - SRpcHead *pNewHead = NULL; - uint8_t * pCont = pHead->content; - SRpcComp *pComp = (SRpcComp *)pHead->content; - - if (pHead->comp) { - // decompress the content - assert(pComp->reserved == 0); - int contLen = htonl(pComp->contLen); - - // prepare the temporary buffer to decompress message - char *temp = (char *)taosMemoryMalloc(contLen + RPC_MSG_OVERHEAD); - pNewHead = (SRpcHead *)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext - - if (pNewHead) { - int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead; - int origLen = LZ4_decompress_safe((char *)(pCont + overhead), (char *)pNewHead->content, compLen, contLen); - assert(origLen == contLen); - - memcpy(pNewHead, pHead, sizeof(SRpcHead)); - pNewHead->msgLen = rpcMsgLenFromCont(origLen); - rpcFreeMsg(pHead); // free the compressed message buffer - pHead = pNewHead; - tTrace("decomp malloc mem:%p", temp); - } else { - tError("failed to allocate memory to decompress msg, contLen:%d", contLen); - } - } - - return pHead; -} - -static int rpcAuthenticateMsg(void *pMsg, int msgLen, void *pAuth, void *pKey) { - T_MD5_CTX context; - int ret = -1; - - tMD5Init(&context); - tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); - tMD5Update(&context, (uint8_t *)pMsg, msgLen); - tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); - tMD5Final(&context); - - if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; - - return ret; -} - -static void rpcBuildAuthHead(void *pMsg, int msgLen, void *pAuth, void *pKey) { - T_MD5_CTX context; - - tMD5Init(&context); - tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); - tMD5Update(&context, (uint8_t *)pMsg, msgLen); - tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); - tMD5Final(&context); - - memcpy(pAuth, context.digest, sizeof(context.digest)); -} - -static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen) { - SRpcHead *pHead = (SRpcHead *)msg; - - if (pConn->spi && pConn->secured == 0) { - // add auth part - pHead->spi = pConn->spi; - SRpcDigest *pDigest = (SRpcDigest *)(msg + msgLen); - pDigest->timeStamp = htonl(taosGetTimestampSec()); - msgLen += sizeof(SRpcDigest); - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - rpcBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); - } else { - pHead->spi = 0; - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - } - - return msgLen; -} - -static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { - SRpcHead *pHead = (SRpcHead *)msg; - int code = 0; - - if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { - // secured link, or no authentication - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - // tTrace("%s, secured link, no auth is required", pConn->info); - return 0; - } - - if (!rpcIsReq(pHead->msgType)) { - // for response, if code is auth failure, it shall bypass the auth process - code = htonl(pHead->code); - if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || - code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || - code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); - return 0; - } - } - - code = 0; - if (pHead->spi == pConn->spi) { - // authentication - SRpcDigest *pDigest = (SRpcDigest *)((char *)pHead + msgLen - sizeof(SRpcDigest)); - - int32_t delta; - delta = (int32_t)htonl(pDigest->timeStamp); - delta -= (int32_t)taosGetTimestampSec(); - if (abs(delta) > 900) { - tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); - code = TSDB_CODE_RPC_INVALID_TIME_STAMP; - } else { - if (rpcAuthenticateMsg(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { - tDebug("%s, authentication failed, msg discarded", pConn->info); - code = TSDB_CODE_RPC_AUTH_FAILURE; - } else { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); - if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client - // tTrace("%s, message is authenticated", pConn->info); - } - } - } else { - tError("%s, auth spi:%d not matched with received:%d %p", pConn->info, pConn->spi, pHead->spi, pConn); - code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED; - } - - return code; -} - -static void rpcLockConn(SRpcConn *pConn) { - int64_t tid = taosGetSelfPthreadId(); - int i = 0; - while (atomic_val_compare_exchange_64(&(pConn->lockedBy), 0, tid) != 0) { - if (++i % 1000 == 0) { - sched_yield(); - } - } -} - -static void rpcUnlockConn(SRpcConn *pConn) { - int64_t tid = taosGetSelfPthreadId(); - if (atomic_val_compare_exchange_64(&(pConn->lockedBy), tid, 0) != tid) { - assert(false); - } -} - -static void rpcAddRef(SRpcInfo *pRpc) { atomic_add_fetch_32(&pRpc->refCount, 1); } - -static void rpcDecRef(SRpcInfo *pRpc) { - if (atomic_sub_fetch_32(&pRpc->refCount, 1) == 0) { - rpcCloseConnCache(pRpc->pCache); - taosHashCleanup(pRpc->hash); - taosTmrCleanUp(pRpc->tmrCtrl); - taosIdPoolCleanUp(pRpc->idPool); - - taosMemoryFreeClear(pRpc->connList); - taosThreadMutexDestroy(&pRpc->mutex); - tDebug("%s rpc resources are released", pRpc->label); - taosMemoryFreeClear(pRpc); - - atomic_sub_fetch_32(&tsRpcNum, 1); - } -} -#endif diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c deleted file mode 100644 index 52c5ddcf631e9a0cc3a307fab00122c5139fa5dd..0000000000000000000000000000000000000000 --- a/source/libs/transport/src/rpcTcp.c +++ /dev/null @@ -1,657 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "rpcTcp.h" -#include "os.h" -#include "rpcHead.h" -#include "rpcLog.h" -#include "taosdef.h" -#include "taoserror.h" -#include "tutil.h" - -#ifndef USE_UV -typedef struct SFdObj { - void * signature; - TdSocketPtr pSocket; // TCP socket FD - void * thandle; // handle from upper layer, like TAOS - uint32_t ip; - uint16_t port; - int16_t closedByApp; // 1: already closed by App - struct SThreadObj *pThreadObj; - struct SFdObj * prev; - struct SFdObj * next; -} SFdObj; - -typedef struct SThreadObj { - TdThread thread; - SFdObj * pHead; - TdThreadMutex mutex; - uint32_t ip; - bool stop; - TdEpollPtr pEpoll; - int numOfFds; - int threadId; - char label[TSDB_LABEL_LEN]; - void * shandle; // handle passed by upper layer during server initialization - void *(*processData)(SRecvInfo *pPacket); -} SThreadObj; - -typedef struct { - char label[TSDB_LABEL_LEN]; - int32_t index; - int numOfThreads; - SThreadObj **pThreadObj; -} SClientObj; - -typedef struct { - TdSocketServerPtr pSocketServer; - uint32_t ip; - uint16_t port; - int8_t stop; - int8_t reserve; - char label[TSDB_LABEL_LEN]; - int numOfThreads; - void * shandle; - SThreadObj **pThreadObj; - TdThread thread; -} SServerObj; - -static void * taosProcessTcpData(void *param); -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket); -static void taosFreeFdObj(SFdObj *pFdObj); -static void taosReportBrokenLink(SFdObj *pFdObj); -static void * taosAcceptTcpConnection(void *arg); - -void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - SServerObj *pServerObj; - SThreadObj *pThreadObj; - - pServerObj = (SServerObj *)taosMemoryCalloc(sizeof(SServerObj), 1); - if (pServerObj == NULL) { - tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - pServerObj->pSocketServer = NULL; - taosResetPthread(&pServerObj->thread); - pServerObj->ip = ip; - pServerObj->port = port; - tstrncpy(pServerObj->label, label, sizeof(pServerObj->label)); - pServerObj->numOfThreads = numOfThreads; - - pServerObj->pThreadObj = (SThreadObj **)taosMemoryCalloc(sizeof(SThreadObj *), numOfThreads); - if (pServerObj->pThreadObj == NULL) { - tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); - taosMemoryFree(pServerObj); - return NULL; - } - - int code = 0; - TdThreadAttr thattr; - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - // initialize parameters in case it may encounter error later - for (int i = 0; i < numOfThreads; ++i) { - pThreadObj = (SThreadObj *)taosMemoryCalloc(sizeof(SThreadObj), 1); - if (pThreadObj == NULL) { - tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); - for (int j = 0; j < i; ++j) taosMemoryFree(pServerObj->pThreadObj[j]); - taosMemoryFree(pServerObj->pThreadObj); - taosMemoryFree(pServerObj); - return NULL; - } - - pServerObj->pThreadObj[i] = pThreadObj; - pThreadObj->pEpoll = NULL; - taosResetPthread(&pThreadObj->thread); - pThreadObj->processData = fp; - tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); - pThreadObj->shandle = shandle; - pThreadObj->stop = false; - } - - // initialize mutex, thread, fd which may fail - for (int i = 0; i < numOfThreads; ++i) { - pThreadObj = pServerObj->pThreadObj[i]; - code = taosThreadMutexInit(&(pThreadObj->mutex), NULL); - if (code < 0) { - tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); - break; - } - - pThreadObj->pEpoll = taosCreateEpoll(10); // size does not matter - if (pThreadObj->pEpoll == NULL) { - tError("%s failed to create TCP epoll", label); - code = -1; - break; - } - - code = taosThreadCreate(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); - if (code != 0) { - tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); - break; - } - - pThreadObj->threadId = i; - } - - pServerObj->pSocketServer = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); - if (pServerObj->pSocketServer == NULL) code = -1; - - if (code == 0) { - code = taosThreadCreate(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); - if (code != 0) { - tError("%s failed to create TCP accept thread(%s)", label, strerror(code)); - } - } - - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - taosCleanUpTcpServer(pServerObj); - pServerObj = NULL; - } else { - tDebug("%s TCP server is initialized, ip:0x%x port:%hu numOfThreads:%d", label, ip, port, numOfThreads); - } - - taosThreadAttrDestroy(&thattr); - return (void *)pServerObj; -} - -static void taosStopTcpThread(SThreadObj *pThreadObj) { - if (pThreadObj == NULL) { - return; - } - // save thread into local variable and signal thread to stop - TdThread thread = pThreadObj->thread; - if (!taosCheckPthreadValid(thread)) { - return; - } - pThreadObj->stop = true; - if (taosComparePthread(thread, taosThreadSelf())) { - pthread_detach(taosThreadSelf()); - return; - } - taosThreadJoin(thread, NULL); -} - -void taosStopTcpServer(void *handle) { - SServerObj *pServerObj = handle; - - if (pServerObj == NULL) return; - pServerObj->stop = 1; - - if (pServerObj->pSocketServer != NULL) { - taosShutDownSocketServerRD(pServerObj->pSocketServer); - } - if (taosCheckPthreadValid(pServerObj->thread)) { - if (taosComparePthread(pServerObj->thread, taosThreadSelf())) { - pthread_detach(taosThreadSelf()); - } else { - taosThreadJoin(pServerObj->thread, NULL); - } - } - - tDebug("%s TCP server is stopped", pServerObj->label); -} - -void taosCleanUpTcpServer(void *handle) { - SServerObj *pServerObj = handle; - SThreadObj *pThreadObj; - if (pServerObj == NULL) return; - - for (int i = 0; i < pServerObj->numOfThreads; ++i) { - pThreadObj = pServerObj->pThreadObj[i]; - taosStopTcpThread(pThreadObj); - } - - tDebug("%s TCP server is cleaned up", pServerObj->label); - - taosMemoryFreeClear(pServerObj->pThreadObj); - taosMemoryFreeClear(pServerObj); -} - -static void *taosAcceptTcpConnection(void *arg) { - TdSocketPtr pSocket = NULL; - struct sockaddr_in caddr; - int threadId = 0; - SThreadObj * pThreadObj; - SServerObj * pServerObj; - - pServerObj = (SServerObj *)arg; - tDebug("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); - setThreadName("acceptTcpConn"); - - while (1) { - socklen_t addrlen = sizeof(caddr); - pSocket = taosAcceptTcpConnectSocket(pServerObj->pSocketServer, (struct sockaddr *)&caddr, &addrlen); - if (pServerObj->stop) { - tDebug("%s TCP server stop accepting new connections", pServerObj->label); - break; - } - - if (pSocket == NULL) { - if (errno == EINVAL) { - tDebug("%s TCP server stop accepting new connections, exiting", pServerObj->label); - break; - } - - tError("%s TCP accept failure(%s)", pServerObj->label, strerror(errno)); - continue; - } - - taosKeepTcpAlive(pSocket); - struct timeval to = {5, 0}; - int32_t ret = taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); - if (ret != 0) { - taosCloseSocket(&pSocket); - tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno), - taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); - continue; - } - - // pick up the thread to handle this connection - pThreadObj = pServerObj->pThreadObj[threadId]; - - SFdObj *pFdObj = taosMallocFdObj(pThreadObj, pSocket); - if (pFdObj) { - pFdObj->ip = caddr.sin_addr.s_addr; - pFdObj->port = htons(caddr.sin_port); - tDebug("%s new TCP connection from %s:%hu, FD:%p numOfFds:%d", pServerObj->label, - taosInetNtoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds); - } else { - taosCloseSocket(&pSocket); - tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), - taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); - } - - // pick up next thread for next connection - threadId++; - threadId = threadId % pServerObj->numOfThreads; - } - - taosCloseSocketServer(&pServerObj->pSocketServer); - return NULL; -} - -void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - SClientObj *pClientObj = (SClientObj *)taosMemoryCalloc(1, sizeof(SClientObj)); - if (pClientObj == NULL) { - tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - tstrncpy(pClientObj->label, label, sizeof(pClientObj->label)); - pClientObj->numOfThreads = numOfThreads; - pClientObj->pThreadObj = (SThreadObj **)taosMemoryCalloc(numOfThreads, sizeof(SThreadObj *)); - if (pClientObj->pThreadObj == NULL) { - tError("TCP:%s no enough memory", label); - taosMemoryFreeClear(pClientObj); - terrno = TAOS_SYSTEM_ERROR(errno); - } - - int code = 0; - TdThreadAttr thattr; - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - for (int i = 0; i < numOfThreads; ++i) { - SThreadObj *pThreadObj = (SThreadObj *)taosMemoryCalloc(1, sizeof(SThreadObj)); - if (pThreadObj == NULL) { - tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); - for (int j = 0; j < i; ++j) taosMemoryFree(pClientObj->pThreadObj[j]); - taosMemoryFree(pClientObj); - taosThreadAttrDestroy(&thattr); - return NULL; - } - pClientObj->pThreadObj[i] = pThreadObj; - taosResetPthread(&pThreadObj->thread); - pThreadObj->ip = ip; - pThreadObj->stop = false; - tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); - pThreadObj->shandle = shandle; - pThreadObj->processData = fp; - } - - // initialize mutex, thread, fd which may fail - for (int i = 0; i < numOfThreads; ++i) { - SThreadObj *pThreadObj = pClientObj->pThreadObj[i]; - code = taosThreadMutexInit(&(pThreadObj->mutex), NULL); - if (code < 0) { - tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); - break; - } - - pThreadObj->pEpoll = taosCreateEpoll(10); // size does not matter - if (pThreadObj->pEpoll == NULL) { - tError("%s failed to create TCP epoll", label); - code = -1; - break; - } - - code = taosThreadCreate(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); - if (code != 0) { - tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); - break; - } - pThreadObj->threadId = i; - } - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - taosCleanUpTcpClient(pClientObj); - pClientObj = NULL; - } - return pClientObj; -} - -void taosStopTcpClient(void *chandle) { - SClientObj *pClientObj = chandle; - - if (pClientObj == NULL) return; - - tDebug("%s TCP client is stopped", pClientObj->label); -} - -void taosCleanUpTcpClient(void *chandle) { - SClientObj *pClientObj = chandle; - if (pClientObj == NULL) return; - for (int i = 0; i < pClientObj->numOfThreads; ++i) { - SThreadObj *pThreadObj = pClientObj->pThreadObj[i]; - taosStopTcpThread(pThreadObj); - } - - tDebug("%s TCP client is cleaned up", pClientObj->label); - taosMemoryFreeClear(pClientObj->pThreadObj); - taosMemoryFreeClear(pClientObj); -} - -void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { - SClientObj *pClientObj = shandle; - int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; - atomic_store_32(&pClientObj->index, index + 1); - SThreadObj *pThreadObj = pClientObj->pThreadObj[index]; - - TdSocketPtr pSocket = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); - if (pSocket == NULL) return NULL; - - struct sockaddr_in sin; - uint16_t localPort = 0; - unsigned int addrlen = sizeof(sin); - if (taosGetSocketName(pSocket, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { - localPort = (uint16_t)ntohs(sin.sin_port); - } - - SFdObj *pFdObj = taosMallocFdObj(pThreadObj, pSocket); - - if (pFdObj) { - pFdObj->thandle = thandle; - pFdObj->port = port; - pFdObj->ip = ip; - - char ipport[40] = {0}; - taosIpPort2String(ip, port, ipport); - tDebug("%s %p TCP connection to %s is created, localPort:%hu FD:%p numOfFds:%d", pThreadObj->label, thandle, - ipport, localPort, pFdObj, pThreadObj->numOfFds); - } else { - tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); - taosCloseSocket(&pSocket); - } - - return pFdObj; -} - -void taosCloseTcpConnection(void *chandle) { - SFdObj *pFdObj = chandle; - if (pFdObj == NULL || pFdObj->signature != pFdObj) return; - - SThreadObj *pThreadObj = pFdObj->pThreadObj; - tDebug("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj); - - // pFdObj->thandle = NULL; - pFdObj->closedByApp = 1; - taosShutDownSocketWR(pFdObj->pSocket); -} - -int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { - SFdObj *pFdObj = chandle; - if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1; - SThreadObj *pThreadObj = pFdObj->pThreadObj; - - int ret = taosWriteMsg(pFdObj->pSocket, data, len); - tTrace("%s %p TCP data is sent, FD:%p bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, ret); - - return ret; -} - -static void taosReportBrokenLink(SFdObj *pFdObj) { - SThreadObj *pThreadObj = pFdObj->pThreadObj; - - // notify the upper layer, so it will clean the associated context - if (pFdObj->closedByApp == 0) { - taosShutDownSocketWR(pFdObj->pSocket); - - SRecvInfo recvInfo; - recvInfo.msg = NULL; - recvInfo.msgLen = 0; - recvInfo.ip = 0; - recvInfo.port = 0; - recvInfo.shandle = pThreadObj->shandle; - recvInfo.thandle = pFdObj->thandle; - recvInfo.chandle = NULL; - recvInfo.connType = RPC_CONN_TCP; - (*(pThreadObj->processData))(&recvInfo); - } - - taosFreeFdObj(pFdObj); -} - -static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { - SRpcHead rpcHead; - int32_t msgLen, leftLen, retLen, headLen; - char * buffer, *msg; - - SThreadObj *pThreadObj = pFdObj->pThreadObj; - - headLen = taosReadMsg(pFdObj->pSocket, &rpcHead, sizeof(SRpcHead)); - if (headLen != sizeof(SRpcHead)) { - tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen); - return -1; - } - - msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); - int32_t size = msgLen + tsRpcOverhead; - buffer = taosMemoryMalloc(size); - if (NULL == buffer) { - tError("%s %p TCP taosMemoryMalloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); - return -1; - } else { - tTrace("%s %p read data, FD:%p TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, buffer); - } - - msg = buffer + tsRpcOverhead; - leftLen = msgLen - headLen; - retLen = taosReadMsg(pFdObj->pSocket, msg + headLen, leftLen); - - if (leftLen != retLen) { - tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); - taosMemoryFree(buffer); - return -1; - } - - memcpy(msg, &rpcHead, sizeof(SRpcHead)); - - pInfo->msg = msg; - pInfo->msgLen = msgLen; - pInfo->ip = pFdObj->ip; - pInfo->port = pFdObj->port; - pInfo->shandle = pThreadObj->shandle; - pInfo->thandle = pFdObj->thandle; - pInfo->chandle = pFdObj; - pInfo->connType = RPC_CONN_TCP; - - if (pFdObj->closedByApp) { - taosMemoryFree(buffer); - return -1; - } - - return 0; -} - -#define maxEvents 10 - -static void *taosProcessTcpData(void *param) { - SThreadObj * pThreadObj = param; - SFdObj * pFdObj; - struct epoll_event events[maxEvents]; - SRecvInfo recvInfo; - - char name[16] = {0}; - snprintf(name, tListLen(name), "%s-tcp", pThreadObj->label); - setThreadName(name); - - while (1) { - int fdNum = taosWaitEpoll(pThreadObj->pEpoll, events, maxEvents, TAOS_EPOLL_WAIT_TIME); - if (pThreadObj->stop) { - tDebug("%s TCP thread get stop event, exiting...", pThreadObj->label); - break; - } - if (fdNum < 0) continue; - - for (int i = 0; i < fdNum; ++i) { - pFdObj = events[i].data.ptr; - - if (events[i].events & EPOLLERR) { - tDebug("%s %p FD:%p epoll errors", pThreadObj->label, pFdObj->thandle, pFdObj); - taosReportBrokenLink(pFdObj); - continue; - } - - if (events[i].events & EPOLLRDHUP) { - tDebug("%s %p FD:%p RD hang up", pThreadObj->label, pFdObj->thandle, pFdObj); - taosReportBrokenLink(pFdObj); - continue; - } - - if (events[i].events & EPOLLHUP) { - tDebug("%s %p FD:%p hang up", pThreadObj->label, pFdObj->thandle, pFdObj); - taosReportBrokenLink(pFdObj); - continue; - } - - if (taosReadTcpData(pFdObj, &recvInfo) < 0) { - taosShutDownSocketWR(pFdObj->pSocket); - continue; - } - - pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo); - if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj); - } - - if (pThreadObj->stop) break; - } - - if (pThreadObj->pEpoll != NULL) { - taosCloseEpoll(&pThreadObj->pEpoll); - pThreadObj->pEpoll = NULL; - } - - while (pThreadObj->pHead) { - pFdObj = pThreadObj->pHead; - pThreadObj->pHead = pFdObj->next; - taosReportBrokenLink(pFdObj); - } - - taosThreadMutexDestroy(&(pThreadObj->mutex)); - tDebug("%s TCP thread exits ...", pThreadObj->label); - taosMemoryFreeClear(pThreadObj); - - return NULL; -} - -static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { - struct epoll_event event; - - SFdObj *pFdObj = (SFdObj *)taosMemoryCalloc(sizeof(SFdObj), 1); - if (pFdObj == NULL) { - return NULL; - } - - pFdObj->closedByApp = 0; - pFdObj->pSocket = pSocket; - pFdObj->pThreadObj = pThreadObj; - pFdObj->signature = pFdObj; - - event.events = EPOLLIN | EPOLLRDHUP; - event.data.ptr = pFdObj; - if (taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_ADD, pSocket, &event) < 0) { - taosMemoryFreeClear(pFdObj); - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - // notify the data process, add into the FdObj list - taosThreadMutexLock(&(pThreadObj->mutex)); - pFdObj->next = pThreadObj->pHead; - if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj; - pThreadObj->pHead = pFdObj; - pThreadObj->numOfFds++; - taosThreadMutexUnlock(&(pThreadObj->mutex)); - - return pFdObj; -} - -static void taosFreeFdObj(SFdObj *pFdObj) { - if (pFdObj == NULL) return; - if (pFdObj->signature != pFdObj) return; - - SThreadObj *pThreadObj = pFdObj->pThreadObj; - taosThreadMutexLock(&pThreadObj->mutex); - - if (pFdObj->signature == NULL) { - taosThreadMutexUnlock(&pThreadObj->mutex); - return; - } - - pFdObj->signature = NULL; - taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_DEL, pFdObj->pSocket, NULL); - taosCloseSocket(&pFdObj->pSocket); - - pThreadObj->numOfFds--; - if (pThreadObj->numOfFds < 0) - tError("%s %p TCP thread:%d, number of FDs is negative!!!", pThreadObj->label, pFdObj->thandle, - pThreadObj->threadId); - - if (pFdObj->prev) { - (pFdObj->prev)->next = pFdObj->next; - } else { - pThreadObj->pHead = pFdObj->next; - } - - if (pFdObj->next) { - (pFdObj->next)->prev = pFdObj->prev; - } - - taosThreadMutexUnlock(&pThreadObj->mutex); - - tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); - - taosMemoryFreeClear(pFdObj); -} -#endif \ No newline at end of file diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c deleted file mode 100644 index 359a94011dc493493aa9c63660806787b3be4d11..0000000000000000000000000000000000000000 --- a/source/libs/transport/src/rpcUdp.c +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "rpcUdp.h" -#include "os.h" -#include "rpcHead.h" -#include "rpcLog.h" -#include "taosdef.h" -#include "taoserror.h" -#include "ttimer.h" -#include "tutil.h" - -#ifndef USE_UV - -#define RPC_MAX_UDP_CONNS 256 -#define RPC_MAX_UDP_PKTS 1000 -#define RPC_UDP_BUF_TIME 5 // mseconds -#define RPC_MAX_UDP_SIZE 65480 - -typedef struct { - int index; - TdSocketPtr pSocket; - uint16_t port; // peer port - uint16_t localPort; // local port - char label[TSDB_LABEL_LEN]; // copy from udpConnSet; - TdThread thread; - void *hash; - void *shandle; // handle passed by upper layer during server initialization - void *pSet; - void *(*processData)(SRecvInfo *pRecv); - char *buffer; // buffer to receive data -} SUdpConn; - -typedef struct { - int index; - int server; - uint32_t ip; // local IP - uint16_t port; // local Port - void * shandle; // handle passed by upper layer during server initialization - int threads; - char label[TSDB_LABEL_LEN]; - void *(*fp)(SRecvInfo *pPacket); - SUdpConn udpConn[]; -} SUdpConnSet; - -static void *taosRecvUdpData(void *param); - -void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { - SUdpConn * pConn; - SUdpConnSet *pSet; - - int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn); - pSet = (SUdpConnSet *)taosMemoryMalloc((size_t)size); - if (pSet == NULL) { - tError("%s failed to allocate UdpConn", label); - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - memset(pSet, 0, (size_t)size); - pSet->ip = ip; - pSet->port = port; - pSet->shandle = shandle; - pSet->fp = fp; - pSet->threads = threads; - tstrncpy(pSet->label, label, sizeof(pSet->label)); - - TdThreadAttr thAttr; - taosThreadAttrInit(&thAttr); - taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - - int i; - uint16_t ownPort; - for (i = 0; i < threads; ++i) { - pConn = pSet->udpConn + i; - ownPort = (port ? port + i : 0); - pConn->pSocket = taosOpenUdpSocket(ip, ownPort); - if (pConn->pSocket == NULL) { - tError("%s failed to open UDP socket %x:%hu", label, ip, port); - break; - } - - pConn->buffer = taosMemoryMalloc(RPC_MAX_UDP_SIZE); - if (NULL == pConn->buffer) { - tError("%s failed to malloc recv buffer", label); - break; - } - - struct sockaddr_in sin; - unsigned int addrlen = sizeof(sin); - if (taosGetSocketName(pConn->pSocket, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && - addrlen == sizeof(sin)) { - pConn->localPort = (uint16_t)ntohs(sin.sin_port); - } - - tstrncpy(pConn->label, label, sizeof(pConn->label)); - pConn->shandle = shandle; - pConn->processData = fp; - pConn->index = i; - pConn->pSet = pSet; - - int code = taosThreadCreate(&pConn->thread, &thAttr, taosRecvUdpData, pConn); - if (code != 0) { - tError("%s failed to create thread to process UDP data(%s)", label, strerror(errno)); - break; - } - } - - taosThreadAttrDestroy(&thAttr); - - if (i != threads) { - terrno = TAOS_SYSTEM_ERROR(errno); - taosCleanUpUdpConnection(pSet); - return NULL; - } - - tDebug("%s UDP connection is initialized, ip:%x:%hu threads:%d", label, ip, port, threads); - return pSet; -} - -void taosStopUdpConnection(void *handle) { - SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn * pConn; - - if (pSet == NULL) return; - - for (int i = 0; i < pSet->threads; ++i) { - pConn = pSet->udpConn + i; - if (pConn->pSocket != NULL) taosShutDownSocketRDWR(pConn->pSocket); - if (pConn->pSocket != NULL) taosCloseSocket(&pConn->pSocket); - pConn->pSocket = NULL; - } - - for (int i = 0; i < pSet->threads; ++i) { - pConn = pSet->udpConn + i; - if (taosCheckPthreadValid(pConn->thread)) { - taosThreadJoin(pConn->thread, NULL); - } - taosMemoryFreeClear(pConn->buffer); - // tTrace("%s UDP thread is closed, index:%d", pConn->label, i); - } - - tDebug("%s UDP is stopped", pSet->label); -} - -void taosCleanUpUdpConnection(void *handle) { - SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn * pConn; - - if (pSet == NULL) return; - - for (int i = 0; i < pSet->threads; ++i) { - pConn = pSet->udpConn + i; - if (pConn->pSocket != NULL) taosCloseSocket(&pConn->pSocket); - } - - tDebug("%s UDP is cleaned up", pSet->label); - taosMemoryFreeClear(pSet); -} - -void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { - SUdpConnSet *pSet = (SUdpConnSet *)shandle; - - pSet->index = (pSet->index + 1) % pSet->threads; - - SUdpConn *pConn = pSet->udpConn + pSet->index; - pConn->port = port; - - tDebug("%s UDP connection is setup, ip:%x:%hu localPort:%hu", pConn->label, ip, port, pConn->localPort); - - return pConn; -} - -static void *taosRecvUdpData(void *param) { - SUdpConn * pConn = param; - struct sockaddr_in sourceAdd; - ssize_t dataLen; - unsigned int addLen; - uint16_t port; - SRecvInfo recvInfo; - - memset(&sourceAdd, 0, sizeof(sourceAdd)); - addLen = sizeof(sourceAdd); - tDebug("%s UDP thread is created, index:%d", pConn->label, pConn->index); - char *msg = pConn->buffer; - - setThreadName("recvUdpData"); - - while (1) { - dataLen = taosReadFromSocket(pConn->pSocket, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); - if (dataLen <= 0) { - tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d", pConn->label, strerror(errno), (int32_t)dataLen); - - // for windows usage, remote shutdown also returns - 1 in windows client - if (pConn->pSocket == NULL) { - break; - } else { - continue; - } - } - - port = ntohs(sourceAdd.sin_port); - - if (dataLen < sizeof(SRpcHead)) { - tError("%s recvfrom failed(%s)", pConn->label, strerror(errno)); - continue; - } - - int32_t size = dataLen + tsRpcOverhead; - char * tmsg = taosMemoryMalloc(size); - if (NULL == tmsg) { - tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen); - continue; - } else { - tTrace("UDP malloc mem:%p size:%d", tmsg, size); - } - - tmsg += tsRpcOverhead; // overhead for SRpcReqContext - memcpy(tmsg, msg, dataLen); - recvInfo.msg = tmsg; - recvInfo.msgLen = dataLen; - recvInfo.ip = sourceAdd.sin_addr.s_addr; - recvInfo.port = port; - recvInfo.shandle = pConn->shandle; - recvInfo.thandle = NULL; - recvInfo.chandle = pConn; - recvInfo.connType = 0; - (*(pConn->processData))(&recvInfo); - } - - return NULL; -} - -int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) { - SUdpConn *pConn = (SUdpConn *)chandle; - - if (pConn == NULL) return -1; - - struct sockaddr_in destAdd; - memset(&destAdd, 0, sizeof(destAdd)); - destAdd.sin_family = AF_INET; - destAdd.sin_addr.s_addr = ip; - destAdd.sin_port = htons(port); - - int ret = taosSendto(pConn->pSocket, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd)); - - return ret; -} -#endif \ No newline at end of file diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index da83a6f37fc5b03cc880165d25689c918963ec7f..36f5cf98150e5636b43eb35b819d5bcd9288fe6a 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -295,14 +295,14 @@ static void uvHandleReq(SSrvConn* pConn) { // no ref here } - // if pHead->noResp = 1, + // pHead->noResp = 1, // 1. server application should not send resp on handle // 2. once send out data, cli conn released to conn pool immediately // 3. not mixed with persist transMsg.info.handle = (void*)uvAcquireExHandle(pConn->refId); - tTrace("server handle %p conn: %p translated to app, refId: %" PRIu64 "", transMsg.info.handle, pConn, pConn->refId); transMsg.info.refId = pConn->refId; + tTrace("server handle %p conn: %p translated to app, refId: %" PRIu64 "", transMsg.info.handle, pConn, pConn->refId); assert(transMsg.info.handle != NULL); if (pHead->noResp == 1) { transMsg.info.refId = -1; diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 2bf086b99bbdb5cc4709724ba4bba7d7e9dfcbba..8b1dcd46cfc0143c7d52d744317592b70eb5ec19 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -15,9 +15,9 @@ //#define _DEFAULT_SOURCE #include "os.h" -#include "rpcLog.h" #include "tglobal.h" #include "tqueue.h" +#include "transLog.h" #include "trpc.h" int msgSize = 128; diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 5755e4a273bbc654b222ed23c20e8fc14e73b5a9..55e6dd000a1270889894fe28f0a024164e5255eb 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -14,9 +14,9 @@ */ #include #include "os.h" -#include "rpcLog.h" #include "taoserror.h" #include "tglobal.h" +#include "transLog.h" #include "trpc.h" #include "tutil.h" @@ -161,7 +161,7 @@ int main(int argc, char *argv[]) { } } - const char *path = "/tmp/transport/client"; + const char *path = TD_TMP_DIR_PATH "transport/client"; taosRemoveDir(path); taosMkDir(path); tstrncpy(tsLogDir, path, PATH_MAX); diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 42bebe5191801ad2e451c2bb4cea7b573aeeecb4..1fd78be77de7575d47e3d2ce70bf6e0908dfec8e 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -15,9 +15,9 @@ //#define _DEFAULT_SOURCE #include "os.h" -#include "rpcLog.h" #include "tglobal.h" #include "tqueue.h" +#include "transLog.h" #include "trpc.h" int msgSize = 128; @@ -160,7 +160,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - const char *path = "/tmp/transport/server"; + const char *path = TD_TMP_DIR_PATH "transport/server"; taosRemoveDir(path); taosMkDir(path); tstrncpy(tsLogDir, path, PATH_MAX); diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index 6fb7d81fcab1ae1e0db6814758f95046359b89e3..bc6461eaed1f5d2c7ca9d82f9ae47dbebf4fc02b 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -14,9 +14,9 @@ */ #include #include "os.h" -#include "rpcLog.h" #include "taoserror.h" #include "tglobal.h" +#include "transLog.h" #include "trpc.h" #include "tutil.h" diff --git a/source/libs/transport/test/transUT.cpp b/source/libs/transport/test/transUT.cpp index 4829f5aa397fd96dfb05caf23023b6a35d852e26..25b04e769cfe248046fc8e080d1775c331ddcdcd 100644 --- a/source/libs/transport/test/transUT.cpp +++ b/source/libs/transport/test/transUT.cpp @@ -15,10 +15,10 @@ #include #include #include -#include "rpcLog.h" #include "tdatablock.h" #include "tglobal.h" #include "tlog.h" +#include "transLog.h" #include "trpc.h" using namespace std; @@ -43,7 +43,7 @@ static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); class Client { public: void Init(int nThread) { - memcpy(tsTempDir, "/tmp", strlen("/tmp")); + memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH)); memset(&rpcInit_, 0, sizeof(rpcInit_)); rpcInit_.localPort = 0; rpcInit_.label = (char *)label; @@ -105,7 +105,7 @@ class Client { class Server { public: Server() { - memcpy(tsTempDir, "/tmp", strlen("/tmp")); + memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH)); memset(&rpcInit_, 0, sizeof(rpcInit_)); memcpy(rpcInit_.localFqdn, "localhost", strlen("localhost")); @@ -219,7 +219,7 @@ static void initEnv() { tsLogEmbedded = 1; tsAsyncLog = 0; - std::string path = "/tmp/transport"; + std::string path = TD_TMP_DIR_PATH "transport"; // taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 2ddcb27835b5276026d46ff2d46150b7fc9995a7..d2a43c410708249983295dca44ca06f6f75a2b70 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -121,6 +121,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { pWal->vers.lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; + taosCloseFile(&pIdxTFile); + taosCloseFile(&pLogTFile); // unlock taosThreadMutexUnlock(&pWal->mutex); diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 18345699b2f5fa55d8356cab7808a09837f61a2e..b1c673e87bfe702f2111c48713fd809199f2520e 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -37,7 +37,7 @@ class WalCleanEnv : public ::testing::Test { } SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; }; class WalCleanDeleteEnv : public ::testing::Test { @@ -67,7 +67,7 @@ class WalCleanDeleteEnv : public ::testing::Test { } SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; }; class WalKeepEnv : public ::testing::Test { @@ -104,7 +104,7 @@ class WalKeepEnv : public ::testing::Test { } SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; }; class WalRetentionEnv : public ::testing::Test { @@ -141,7 +141,7 @@ class WalRetentionEnv : public ::testing::Test { } SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; }; TEST_F(WalCleanEnv, createNew) { @@ -325,6 +325,7 @@ TEST_F(WalKeepEnv, readHandleRead) { EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); } } + walCloseReadHandle(pRead); } TEST_F(WalRetentionEnv, repairMeta1) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 11851ca5d8928e58e3080499a64a80ede53976e1..3890a55ff16da11d132261c070b809d833f1b2aa 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -45,32 +45,10 @@ STaosError errors[] = { {.val = 0, .str = "success"}, #endif -// rpc -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ACTION_IN_PROGRESS, "Action in progress") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_REQUIRED, "Authentication required") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, "Authentication failure") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Redirect") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NOT_READY, "System not ready") // peer is not ready to process data -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ALREADY_PROCESSED, "Message already processed") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED, "Last session not finished") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MISMATCHED_LINK_ID, "Mismatched meter id") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TOO_SLOW, "Processing of request timed out") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS, "Number of sessions reached limit") // too many sessions -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_APP_ERROR, "Unexpected generic error in RPC") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_UNEXPECTED_RESPONSE, "Unexpected response") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VALUE, "Invalid value") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TRAN_ID, "Invalid transaction id") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID, "Invalid session id") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, "Invalid message type") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, "Invalid response type") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, "Client and server's time is not synchronized") -TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, "Database not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VERSION, "Invalid app version") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "port already in use") - //common & util +TAOS_DEFINE_ERROR(TSDB_CODE_ACTION_IN_PROGRESS, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_APP_ERROR, "Unexpected generic error") +TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, "Database not ready") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_MEMORY, "Out of Memory") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RANGE, "Out of range") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_SHM_MEM, "Out of Shared memory") @@ -104,6 +82,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, "Invalid Ref ID") TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, "Ref is already there") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, "Ref is not there") +// rpc +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Redirect") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, "Authentication failure") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") + //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_QHANDLE, "Invalid qhandle") @@ -149,7 +134,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt cl // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Mnode not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection") diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 4bbf6ccf450ae4c11685d5012d4acdf2348b551e..551c3b67c8642b8bceab70c9cae75aca78f73769 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -15,8 +15,8 @@ #define _DEFAULT_SOURCE #include "thash.h" -#include "taoserror.h" #include "os.h" +#include "taoserror.h" #include "tlog.h" // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT @@ -27,36 +27,36 @@ #define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) -#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) -#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) -#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) +#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode))) #define FREE_HASH_NODE(_fp, _n) \ do { \ -/* if (_fp != NULL) { \ - (_fp)(_n); \ - }*/ \ + /* if (_fp != NULL) { \ + (_fp)(_n); \ + }*/ \ taosMemoryFreeClear(_n); \ } while (0); struct SHashNode { - SHashNode *next; - uint32_t hashVal; // the hash value of key - uint32_t dataLen; // length of data - uint32_t keyLen; // length of the key - uint16_t refCount; // reference count - int8_t removed; // flag to indicate removed - char data[]; + SHashNode *next; + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + uint16_t refCount; // reference count + int8_t removed; // flag to indicate removed + char data[]; }; typedef struct SHashEntry { - int32_t num; // number of elements in current entry - SRWLatch latch; // entry latch - SHashNode *next; + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SHashNode *next; } SHashEntry; struct SHashObj { - SHashEntry ** hashList; + SHashEntry **hashList; size_t capacity; // number of slots int64_t size; // number of elements in hash table _hash_fn_t hashFp; // hash function @@ -65,7 +65,7 @@ struct SHashObj { SRWLatch lock; // read-write spin lock SHashLockTypeE type; // lock type bool enableUpdate; // enable update - SArray * pMemBlock; // memory block allocated for SHashEntry + SArray *pMemBlock; // memory block allocated for SHashEntry _hash_before_fn_t callbackFp; // function invoked before return the value to caller }; @@ -103,14 +103,14 @@ static FORCE_INLINE void taosHashRUnlock(SHashObj *pHashObj) { taosRUnLockLatch(&pHashObj->lock); } -static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry* pe) { +static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry *pe) { if (pHashObj->type == HASH_NO_LOCK) { return; } taosWLockLatch(&pe->latch); } -static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry* pe) { +static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry *pe) { if (pHashObj->type == HASH_NO_LOCK) { return; } @@ -118,7 +118,7 @@ static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEnt taosWUnLockLatch(&pe->latch); } -static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry* pe) { +static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry *pe) { if (pHashObj->type == HASH_NO_LOCK) { return; } @@ -126,7 +126,7 @@ static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry taosRLockLatch(&pe->latch); } -static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry* pe) { +static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry *pe) { if (pHashObj->type == HASH_NO_LOCK) { return; } @@ -142,12 +142,11 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { return i; } -static FORCE_INLINE SHashNode * -doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { +static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, + uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && - ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; @@ -186,18 +185,22 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param pNode the old node with requested key * @param pNewNode the new node with requested key */ -static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode, + SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); atomic_sub_fetch_16(&pNode->refCount, 1); if (prev != NULL) { prev->next = pNewNode; + ASSERT(prev->next != prev); } else { pe->next = pNewNode; } if (pNode->refCount <= 0) { pNewNode->next = pNode->next; + ASSERT(pNewNode->next != pNewNode); + FREE_HASH_NODE(pHashObj->freeFp, pNode); } else { pNewNode->next = pNode; @@ -227,9 +230,7 @@ static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj); * @param pHashObj * @return */ -static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { - return taosHashGetSize(pHashObj) == 0; -} +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return taosHashGetSize(pHashObj) == 0; } SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { if (fn == NULL) { @@ -251,7 +252,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp pHashObj->capacity = taosHashCapacity((int32_t)capacity); pHashObj->equalFp = memcmp; - pHashObj->hashFp = fn; + pHashObj->hashFp = fn; pHashObj->type = type; pHashObj->enableUpdate = update; @@ -305,7 +306,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { if (pHashObj == NULL) { return 0; } - return (int32_t)atomic_load_64((int64_t*)&pHashObj->size); + return (int32_t)atomic_load_64((int64_t *)&pHashObj->size); } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) { @@ -340,10 +341,9 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo } #endif - SHashNode* prev = NULL; + SHashNode *prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && - (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 && + if ((pNode->keyLen == keyLen) && (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; @@ -391,27 +391,27 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo } } -static void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef); +static void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef); void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { - void* p = NULL; + void *p = NULL; return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, false); } int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf) { terrno = 0; - /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false); + /*char* p = */ taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false); return terrno; } -int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t* size) { +int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t *size) { terrno = 0; - /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false); + /*char* p = */ taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false); return terrno; } -void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef) { +void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } @@ -449,15 +449,15 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** if (size != NULL) { if (*d == NULL) { - *size = pNode->dataLen; + *size = pNode->dataLen; *d = taosMemoryCalloc(1, *size); if (*d == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } } else if (*size < pNode->dataLen) { - *size = pNode->dataLen; - char* tmp = taosMemoryRealloc(*d, *size); + *size = pNode->dataLen; + char *tmp = taosMemoryRealloc(*d, *size); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -508,13 +508,12 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { return -1; } - int code = -1; + int code = -1; SHashNode *pNode = pe->next; SHashNode *prevNode = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && - ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { code = 0; // it is found @@ -525,6 +524,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { pe->next = pNode->next; } else { prevNode->next = pNode->next; + ASSERT(prevNode->next != prevNode); } pe->num--; @@ -598,14 +598,14 @@ void taosHashCleanup(SHashObj *pHashObj) { } // for profile only -int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){ +int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return 0; } int32_t num = 0; - taosHashRLock((SHashObj*) pHashObj); + taosHashRLock((SHashObj *)pHashObj); for (int32_t i = 0; i < pHashObj->size; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; @@ -616,7 +616,7 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){ } } - taosHashRUnlock((SHashObj*) pHashObj); + taosHashRUnlock((SHashObj *)pHashObj); return num; } @@ -627,22 +627,22 @@ void taosHashTableResize(SHashObj *pHashObj) { int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { -// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", -// pHashObj->capacity, HASH_MAX_CAPACITY); + // uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", + // pHashObj->capacity, HASH_MAX_CAPACITY); return; } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { -// uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); + // uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } pHashObj->hashList = pNewEntryList; size_t inc = newCapacity - pHashObj->capacity; - void * p = taosMemoryCalloc(inc, sizeof(SHashEntry)); + void *p = taosMemoryCalloc(inc, sizeof(SHashEntry)); for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); @@ -653,9 +653,9 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->capacity = newCapacity; for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { SHashEntry *pe = pHashObj->hashList[idx]; - SHashNode *pNode; - SHashNode *pNext; - SHashNode *pPrev = NULL; + SHashNode *pNode; + SHashNode *pNext; + SHashNode *pPrev = NULL; if (pe->num == 0) { assert(pe->next == NULL); @@ -688,24 +688,25 @@ void taosHashTableResize(SHashObj *pHashObj) { int64_t et = taosGetTimestampUs(); -// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, -// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); + // uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", + // (int32_t)pHashObj->capacity, + // ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize); + SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize + 1); if (pNewNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pNewNode->keyLen = (uint32_t)keyLen; + pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; pNewNode->dataLen = (uint32_t)dsize; - pNewNode->refCount= 1; + pNewNode->refCount = 1; pNewNode->removed = 0; - pNewNode->next = NULL; + pNewNode->next = NULL; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); @@ -719,6 +720,7 @@ void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { pNode->next = pEntry->next; pEntry->next = pNode; + ASSERT(pNode->next != pNode); pEntry->num += 1; } @@ -727,11 +729,12 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return 0; } - return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void*))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); + return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void *))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + + sizeof(SHashObj); } -void *taosHashGetKey(void *data, size_t* keyLen) { - SHashNode * node = GET_HASH_PNODE(data); +void *taosHashGetKey(void *data, size_t *keyLen) { + SHashNode *node = GET_HASH_PNODE(data); if (keyLen != NULL) { *keyLen = node->keyLen; } @@ -751,8 +754,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { SHashNode *pNode = pe->next; while (pNode) { - if (pNode == pOld) - break; + if (pNode == pOld) break; prevNode = pNode; pNode = pNode->next; @@ -766,11 +768,16 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { } atomic_sub_fetch_16(&pOld->refCount, 1); - if (pOld->refCount <=0) { + if (pOld->refCount <= 0) { if (prevNode) { prevNode->next = pOld->next; + ASSERT(prevNode->next != prevNode); } else { pe->next = pOld->next; + SHashNode* x = pe->next; + if (x != NULL) { + ASSERT(x->next != x); + } } pe->num--; @@ -778,7 +785,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { FREE_HASH_NODE(pHashObj->freeFp, pOld); } } else { -// uError("pNode:%p data:%p is not there!!!", pNode, p); + // uError("pNode:%p data:%p is not there!!!", pNode, p); } return pNode; @@ -787,7 +794,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pHashObj == NULL) return NULL; - int slot = 0; + int slot = 0; char *data = NULL; // only add the read lock to disable the resize process @@ -865,9 +872,9 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) { taosHashRUnlock(pHashObj); } -//TODO remove it +// TODO remove it void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { - void* p = NULL; + void *p = NULL; return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, true); } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 1ad12ef43a0154aa1df6ab748701542cba34aaba..00f123370747fcc29eddbb9ad053514134d3bc8f 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -187,9 +187,6 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { } } else { // NOTE: the size may be -1, the this recycle page has not been flushed to disk yet. size = pg->length; - if (size == -1) { - printf("----\n"); - } } ASSERT(size > 0 || (pg->offset == -1 && pg->length == -1)); diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index cd13ebe8ae3a07db3cd75ec39df3f5795566d1dc..c5d9e830d2ece7229af3d18bd8f4e5613c63854e 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -59,7 +59,7 @@ TEST_F(CfgTest, 02_Basic) { EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0), 0); EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0), 0); EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0), 0); - EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp", 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0), 0); EXPECT_EQ(cfgGetSize(pConfig), 6); @@ -126,7 +126,7 @@ TEST_F(CfgTest, 02_Basic) { EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); EXPECT_STREQ(pItem->name, "test_dir"); - EXPECT_STREQ(pItem->str, "/tmp"); + EXPECT_STREQ(pItem->str, TD_TMP_DIR_PATH); cfgCleanup(pConfig); } diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 5ad3cb42aa1bfd9061a594c3ba412508b6219767..eaf198a483aa5e3e90595d2417516aa53f754331 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -13,7 +13,7 @@ namespace { // simple test void simpleTest() { SDiskbasedBuf* pBuf = NULL; - int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4096, "", "/tmp/"); + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4096, "", TD_TMP_DIR_PATH); int32_t pageId = 0; int32_t groupId = 0; @@ -57,7 +57,7 @@ void simpleTest() { void writeDownTest() { SDiskbasedBuf* pBuf = NULL; - int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/"); + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", TD_TMP_DIR_PATH); int32_t pageId = 0; int32_t writePageId = 0; @@ -106,7 +106,7 @@ void writeDownTest() { void recyclePageTest() { SDiskbasedBuf* pBuf = NULL; - int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/"); + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", TD_TMP_DIR_PATH); int32_t pageId = 0; int32_t writePageId = 0; diff --git a/source/util/test/procTest.cpp b/source/util/test/procTest.cpp index af53ddcea5e705b9e8ae50908ac777bdcce6da47..53d3fa2c4bc118d57d9a8d08ecc4810f83aa5eda 100644 --- a/source/util/test/procTest.cpp +++ b/source/util/test/procTest.cpp @@ -38,9 +38,9 @@ class UtilTesProc : public ::testing::Test { head.noResp = 3; head.persistHandle = 4; - taosRemoveDir("/tmp/td"); - taosMkDir("/tmp/td"); - tstrncpy(tsLogDir, "/tmp/td", PATH_MAX); + taosRemoveDir(TD_TMP_DIR_PATH "td"); + taosMkDir(TD_TMP_DIR_PATH "td"); + tstrncpy(tsLogDir, TD_TMP_DIR_PATH "td", PATH_MAX); if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 952aca70cf5f825f6c3217e53e73e5291b71d83e..9190943dfd25169e9989ce0112242fd046d6e285 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -264,7 +264,7 @@ class TDDnode: cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( binPath, self.cfgDir) else: - valgrindCmdline = "valgrind --log-file=\"valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + valgrindCmdline = "valgrind --log-file=\"%s/../log/valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir cmd = "nohup %s %s -c %s 2>&1 & " % ( valgrindCmdline, binPath, self.cfgDir) @@ -325,7 +325,7 @@ class TDDnode: cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( binPath, self.cfgDir) else: - valgrindCmdline = "valgrind --log-file=\"valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + valgrindCmdline = "valgrind --log-file=\"%s/../log/valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir cmd = "nohup %s %s -c %s 2>&1 & " % ( valgrindCmdline, binPath, self.cfgDir) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index ba1cd00fcb102bf6e0fbcb29c7655a6cc0d7357a..2ded58a979ad16e06f03ab8d4f828f1c10731df3 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -1825,10 +1825,12 @@ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { exit(1); } - - if (taos_stmt_add_batch(stmt)) { - printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); - exit(1); + + if (rand() % 2 == 0) { + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } } if (taos_stmt_execute(stmt) != 0) { @@ -1871,10 +1873,12 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { exit(1); } - - if (taos_stmt_add_batch(stmt)) { - printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); - exit(1); + + if (rand() % 2 == 0) { + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } } if (taos_stmt_execute(stmt) != 0) { diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 18fe3b9afe9776b1d9d0e15767f307265e229ef6..45fa037458b4c415563bb45c62b7a163495b582a 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -28,6 +28,8 @@ ./test.sh -f tsim/insert/basic1.sim ./test.sh -f tsim/insert/backquote.sim ./test.sh -f tsim/insert/null.sim +./test.sh -f tsim/insert/update0.sim +./test.sh -f tsim/insert/commit-merge0.sim # ---- parser ./test.sh -f tsim/parser/groupby-basic.sim @@ -61,9 +63,9 @@ # ---- table ./test.sh -f tsim/table/basic1.sim -# ---- tstream -./test.sh -f tsim/tstream/basic0.sim -./test.sh -f tsim/tstream/basic1.sim +# ---- stream +./test.sh -f tsim/stream/basic0.sim +./test.sh -f tsim/stream/basic1.sim # ---- transaction ./test.sh -f tsim/trans/create_db.sim diff --git a/tests/script/tsim/insert/commit-merge0.sim b/tests/script/tsim/insert/commit-merge0.sim new file mode 100644 index 0000000000000000000000000000000000000000..adbd1904b2071ccc68f978ea1c2da40e208adc93 --- /dev/null +++ b/tests/script/tsim/insert/commit-merge0.sim @@ -0,0 +1,262 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database db days 300 keep 365000d,365000d,365000d +sql show databases +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use db +sql create table stb1(ts timestamp, c6 double) tags (t1 int); +sql create table ct1 using stb1 tags ( 1 ); +sql create table ct2 using stb1 tags ( 2 ); +sql create table ct3 using stb1 tags ( 3 ); +sql create table ct4 using stb1 tags ( 4 ); +sql insert into ct1 values ('2022-05-01 18:30:27.001', 0.0); +sql insert into ct4 values ('2022-04-28 18:30:27.002', 0.0); +sql insert into ct1 values ('2022-05-01 18:30:17.003', 11.11); +sql insert into ct4 values ('2022-02-01 18:30:27.004', 11.11); +sql insert into ct1 values ('2022-05-01 18:30:07.005', 22.22); +sql insert into ct4 values ('2021-11-01 18:30:27.006', 22.22); +sql insert into ct1 values ('2022-05-01 18:29:27.007', 33.33); +sql insert into ct4 values ('2022-08-01 18:30:27.008', 33.33); +sql insert into ct1 values ('2022-05-01 18:20:27.009', 44.44); +sql insert into ct4 values ('2021-05-01 18:30:27.010', 44.44); +sql insert into ct1 values ('2022-05-01 18:21:27.011', 55.55); +sql insert into ct4 values ('2021-01-01 18:30:27.012', 55.55); +sql insert into ct1 values ('2022-05-01 18:22:27.013', 66.66); +sql insert into ct4 values ('2020-06-01 18:30:27.014', 66.66); +sql insert into ct1 values ('2022-05-01 18:28:37.015', 77.77); +sql insert into ct4 values ('2020-05-01 18:30:27.016', 77.77); +sql insert into ct1 values ('2022-05-01 18:29:17.017', 88.88); +sql insert into ct4 values ('2019-05-01 18:30:27.018', 88.88); +sql insert into ct1 values ('2022-05-01 18:30:20.019', 0); +sql insert into ct1 values ('2022-05-01 18:30:47.020', -99.99); +sql insert into ct1 values ('2022-05-01 18:30:49.021', NULL); +sql insert into ct1 values ('2022-05-01 18:30:51.022', -99.99); +sql insert into ct4 values ('2018-05-01 18:30:27.023', NULL) ; +sql insert into ct4 values ('2021-03-01 18:30:27.024', NULL) ; +sql insert into ct4 values ('2022-08-01 18:30:27.025', NULL) ; + +print =============== select * from ct1 - memory +sql select * from stb1; +if $rows != 25 then + print rows = $rows != 25 + return -1 +endi + + +print =============== stop and restart taosd + +$reboot_max = 10; + +$reboot_cnt = 0 + +reboot_and_check: + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +print =============== insert duplicated records to memory - loop $reboot_max - $reboot_cnt +sql use db +sql insert into ct1 values ('2022-05-01 18:30:27.001', 0.0); +sql insert into ct4 values ('2022-04-28 18:30:27.002', 0.0); +sql insert into ct1 values ('2022-05-01 18:30:17.003', 11.11); +sql insert into ct4 values ('2022-02-01 18:30:27.004', 11.11); +sql insert into ct1 values ('2022-05-01 18:30:07.005', 22.22); +sql insert into ct4 values ('2021-11-01 18:30:27.006', 22.22); +sql insert into ct1 values ('2022-05-01 18:29:27.007', 33.33); +sql insert into ct4 values ('2022-08-01 18:30:27.008', 33.33); +sql insert into ct1 values ('2022-05-01 18:20:27.009', 44.44); +sql insert into ct4 values ('2021-05-01 18:30:27.010', 44.44); +sql insert into ct1 values ('2022-05-01 18:21:27.011', 55.55); +sql insert into ct4 values ('2021-01-01 18:30:27.012', 55.55); +sql insert into ct1 values ('2022-05-01 18:22:27.013', 66.66); +sql insert into ct4 values ('2020-06-01 18:30:27.014', 66.66); +sql insert into ct1 values ('2022-05-01 18:28:37.015', 77.77); +sql insert into ct4 values ('2020-05-01 18:30:27.016', 77.77); +sql insert into ct1 values ('2022-05-01 18:29:17.017', 88.88); +sql insert into ct4 values ('2019-05-01 18:30:27.018', 88.88); +sql insert into ct1 values ('2022-05-01 18:30:20.019', 0); +sql insert into ct1 values ('2022-05-01 18:30:47.020', -99.99); +sql insert into ct1 values ('2022-05-01 18:30:49.021', NULL); +sql insert into ct1 values ('2022-05-01 18:30:51.022', -99.99); +sql insert into ct4 values ('2018-05-01 18:30:27.023', NULL) ; +sql insert into ct4 values ('2021-03-01 18:30:27.024', NULL) ; +sql insert into ct4 values ('2022-08-01 18:30:27.025', NULL) ; + +print =============== select * from ct1 - merge memory and file - loop $reboot_max - $reboot_cnt +sql select * from ct1; +if $rows != 13 then + print rows = $rows != 13 + return -1 +endi +print $data[0][0] $data[0][1] +print $data[1][0] $data[1][1] +print $data[2][0] $data[2][1] +print $data[3][0] $data[3][1] +print $data[4][0] $data[4][1] +print $data[5][0] $data[5][1] +print $data[6][0] $data[6][1] +print $data[7][0] $data[7][1] +print $data[8][0] $data[8][1] +print $data[9][0] $data[9][1] +print $data[10][0] $data[10][1] +print $data[11][0] $data[11][1] +print $data[12][0] $data[12][1] + +if $data[0][1] != 44.440000000 then + print $data[0][1] != 44.440000000 + return -1 +endi +if $data[1][1] != 55.550000000 then + print $data[1][1] != 55.550000000 + return -1 +endi +if $data[2][1] != 66.660000000 then + print $data[2][1] != 66.660000000 + return -1 +endi +if $data[3][1] != 77.770000000 then + print $data[3][1] != 77.770000000 + return -1 +endi +if $data[4][1] != 88.880000000 then + print $data[4][1] != 88.880000000 + return -1 +endi +if $data[5][1] != 33.330000000 then + print $data[5][1] != 33.330000000 + return -1 +endi +if $data[6][1] != 22.220000000 then + print $data[6][1] != 22.220000000 + return -1 +endi +if $data[7][1] != 11.110000000 then + print $data[7][1] != 11.110000000 + return -1 +endi +if $data[8][1] != 0.000000000 then + print $data[8][1] != 0.000000000 + return -1 +endi +if $data[9][1] != 0.000000000 then + print $data[9][1] != 0.000000000 + return -1 +endi +if $data[10][1] != -99.990000000 then + print $data[10][1] != -99.990000000 + return -1 +endi +if $data[11][1] != NULL then + print $data[11][1] != NULL + return -1 +endi +if $data[12][1] != -99.990000000 then + print $data[12][1] != -99.990000000 + return -1 +endi + +print =============== select * from ct4 - merge memory and file - loop $reboot_max - $reboot_cnt +sql select * from ct4; +if $rows != 12 then + print rows = $rows != 12 + return -1 +endi + +print $data[0][0] $data[0][1] +print $data[1][0] $data[1][1] +print $data[2][0] $data[2][1] +print $data[3][0] $data[3][1] +print $data[4][0] $data[4][1] +print $data[5][0] $data[5][1] +print $data[6][0] $data[6][1] +print $data[7][0] $data[7][1] +print $data[8][0] $data[8][1] +print $data[9][0] $data[9][1] +print $data[10][0] $data[10][1] +print $data[11][0] $data[11][1] + +if $data[0][1] != NULL then + print $data[0][1] != NULL + return -1 +endi +if $data[1][1] != 88.880000000 then + print $data[1][1] != 88.880000000 + return -1 +endi +if $data[2][1] != 77.770000000 then + print $data[2][1] != 77.770000000 + return -1 +endi +if $data[3][1] != 66.660000000 then + print $data[3][1] != 66.660000000 + return -1 +endi +if $data[4][1] != 55.550000000 then + print $data[4][1] != 55.550000000 + return -1 +endi +if $data[5][1] != NULL then + print $data[5][1] != NULL + return -1 +endi +if $data[6][1] != 44.440000000 then + print $data[6][1] != 44.440000000 + return -1 +endi +if $data[7][1] != 22.220000000 then + print $data[7][1] != 22.220000000 + return -1 +endi +if $data[8][1] != 11.110000000 then + print $data[8][1] != 11.110000000 + return -1 +endi +if $data[9][1] != 0.000000000 then + print $data[9][1] != 0.000000000 + return -1 +endi +if $data[10][1] != 33.330000000 then + print $data[10][1] != 33.330000000 + return -1 +endi +if $data[11][1] != NULL then + print $data[11][1] != NULL + return -1 +endi + + +if $reboot_cnt > $reboot_max then + print reboot_cnt $reboot_cnt > reboot_max $reboot_max + return 0 +else + print reboot_cnt $reboot_cnt <= reboot_max $reboot_max + $reboot_cnt = $reboot_cnt + 1 + goto reboot_and_check +endi diff --git a/tests/script/tsim/insert/update0.sim b/tests/script/tsim/insert/update0.sim index 89eecaf86038ab05e98fc0155c7d839d82f43088..3cb5e4008e3a57e3178721b7e3f5458ef07be52b 100644 --- a/tests/script/tsim/insert/update0.sim +++ b/tests/script/tsim/insert/update0.sim @@ -99,6 +99,23 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + print =============== step3-2 query records of ct1 from file sql select * from ct1; print $data00 $data01 diff --git a/tests/script/tsim/stable/add_column.sim b/tests/script/tsim/stable/add_column.sim index acacc13524f0db3723c8036339bde4a3476208fd..0b2df509f9f4c32f60fd073076517911d1f84f3e 100644 --- a/tests/script/tsim/stable/add_column.sim +++ b/tests/script/tsim/stable/add_column.sim @@ -6,7 +6,7 @@ sql connect print ========== prepare stb and ctb sql create database db vgroups 1 sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" -sql create table db.ctb using db.stb tags(1, 2, "3") +sql create table db.ctb using db.stb tags(101, 102, "103") sql insert into db.ctb values(now, 1, "2") sql show db.stables @@ -62,25 +62,23 @@ endi if $data[0][2] != 2 then return -1 endi -if $data[0][3] != 1 then +if $data[0][3] != 101 then return -1 endi -print ========== add column +print ========== add column c3 sql alter table db.stb add column c3 int -sql alter table db.stb add column c4 bigint -sql alter table db.stb add column c5 binary(12) - sql show db.stables -if $data[0][3] != 6 then +if $data[0][3] != 4 then return -1 endi sql show db.tables -if $data[0][3] != 6 then +if $data[0][3] != 4 then return -1 endi +sql select * from db.stb sql select * from db.stb print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] if $rows != 1 then @@ -95,13 +93,49 @@ endi if $data[0][3] != NULL then return -1 endi -if $data[0][4] != NULL then +if $data[0][4] != 101 then + return -1 +endi + +sql insert into db.ctb values(now+1s, 1, 2, 3) +sql select * from db.stb +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] + +if $rows != 2 then + return -1 +endi +if $data[0][1] != 1 then + return -1 +endi +if $data[0][2] != 2 then + return -1 +endi +if $data[0][3] != NULL then + return -1 +endi +if $data[0][4] != 101 then + return -1 +endi +if $data[1][1] != 1 then return -1 endi -if $data[0][5] != NULL then +if $data[2][2] != 2 then return -1 endi -if $data[0][6] != 1 then +if $data[1][3] != 3 then return -1 endi +if $data[1][4] != 101 then + return -1 +endi + +print ========== add column c4 +sql alter table db.stb add column c4 bigint +sql insert into db.ctb values(now+2s, 1, 2, 3, 4) +sql select * from db.stb +sql select * from db.stb +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] diff --git a/tests/script/tsim/stable/metrics.sim b/tests/script/tsim/stable/metrics.sim index c49de0e8038bc5583f65d6f2e696b5eb9f1a5db5..e68d95511cfd3c4ea556e34ffed5111f05064405 100644 --- a/tests/script/tsim/stable/metrics.sim +++ b/tests/script/tsim/stable/metrics.sim @@ -28,15 +28,17 @@ if $rows != 1 then return -1 endi -print =============== step2 -sql drop table $mt -sql show stables -if $rows != 0 then - return -1 -endi +#TODO OPEN THIS WHEN STABLE DELETE WORKS +#print =============== step2 +#sql drop table $mt +#sql show stables +#if $rows != 0 then +# return -1 +#endi -print =============== step3 -sql create table $mt (ts timestamp, speed int) TAGS(sp int) +#print =============== step3 +#sql create table $mt (ts timestamp, speed int) TAGS(sp int) +#TODO OPEN THIS WHEN STABLE DELETE WORKS sql show stables if $rows != 1 then @@ -134,4 +136,4 @@ if $rows != 2 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tstream/basic0.sim b/tests/script/tsim/stream/basic0.sim similarity index 100% rename from tests/script/tsim/tstream/basic0.sim rename to tests/script/tsim/stream/basic0.sim diff --git a/tests/script/tsim/tstream/basic1.sim b/tests/script/tsim/stream/basic1.sim similarity index 100% rename from tests/script/tsim/tstream/basic1.sim rename to tests/script/tsim/stream/basic1.sim diff --git a/tests/script/tsim/stream/basic2.sim b/tests/script/tsim/stream/basic2.sim new file mode 100644 index 0000000000000000000000000000000000000000..247d8f62eeae7cda38829db5c2c4fd6b2843c787 --- /dev/null +++ b/tests/script/tsim/stream/basic2.sim @@ -0,0 +1,112 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database d0 vgroups 1 +sql show databases +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use d0 + +print =============== create super table, include column type for count/sum/min/max/first +sql create table if not exists stb (ts timestamp, k int) tags (a int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags(1000) +sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) + +sql show tables +if $rows != 3 then + return -1 +endi + +sql create stream s1 trigger at_once into outstb as select _wstartts, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m) + +sql show stables +if $rows != 2 then + return -1 +endi + +print =============== insert data + +sql insert into ct1 values('2022-05-08 03:42:00.000', 234) +sleep 100 + +#=================================================================== +print =============== query data from child table + +sql select `_wstartts`,`min(k)`,`max(k)`,sum_alias from outstb +print rows: $rows +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi + +if $data01 != 234 then + return -1 +endi + +if $data02 != 234 then + return -1 +endi + +if $data03 != 234 then + return -1 +endi + +#=================================================================== +print =============== insert data + +sql insert into ct1 values('2022-05-08 03:57:00.000', -111) +sleep 100 + + +#=================================================================== +print =============== query data from child table + +sql select `_wstartts`,`min(k)`,`max(k)`,sum_alias from outstb +print rows: $rows +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +if $rows != 2 then + return -1 +endi + +if $data01 != 234 then + return -1 +endi + +if $data02 != 234 then + return -1 +endi + +if $data03 != 234 then + return -1 +endi + +if $data11 != -111 then + return -1 +endi + +if $data12 != -111 then + return -1 +endi + +if $data13 != -111 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/basic.sim b/tests/script/tsim/valgrind/basic.sim new file mode 100644 index 0000000000000000000000000000000000000000..0f11ae03134dbc97a2a1ae2464ee9b0aa6e953ea --- /dev/null +++ b/tests/script/tsim/valgrind/basic.sim @@ -0,0 +1,8 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database d0 vgroups 1; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py new file mode 100644 index 0000000000000000000000000000000000000000..964550cdbc8eb5e9a6ce6b1c01884078b68263b8 --- /dev/null +++ b/tests/system-test/0-others/fsync.py @@ -0,0 +1,301 @@ +import datetime + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +PRIMARY_COL = "ts" + +INT_COL = "c1" +BINT_COL = "c2" +SINT_COL = "c3" +TINT_COL = "c4" +FLOAT_COL = "c5" +DOUBLE_COL = "c6" +BOOL_COL = "c7" + +BINARY_COL = "c8" +NCHAR_COL = "c9" +TS_COL = "c10" + +NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +CHAR_COL = [ BINARY_COL, NCHAR_COL, ] +BOOLEAN_COL = [ BOOL_COL, ] +TS_TYPE_COL = [ TS_COL, ] + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def __kill_process(self, process_name): + killCmd = f"ps -ef|grep -w {process_name}| grep -v grep | awk '{{print $2}}' | xargs kill -TERM > /dev/null 2>&1" + + psCmd = f"ps -ef|grep -w {process_name}| grep -v grep | awk '{{print $2}}'" + while processID := subprocess.check_output(psCmd, shell=True): + os.system(killCmd) + time.sleep(1) + + def test_fsync_current(self): + wal_index = 0 + fsync_index = 0 + tdSql.query("show databases") + for i in range(tdSql.queryCols): + if tdSql.cursor.description[i][0] == "wal": + wal_index = i + if tdSql.cursor.description[i][0] == "fsync": + fsync_index = i + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 wal 1") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, wal_index, 1) + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 wal 2") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 fsync 0") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 0) + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 fsync 3000") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 fsync 180000") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 180000) + + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 wal 1 fsync 6000") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 6000) + tdSql.checkData(i, wal_index, 1) + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 wal 2 fsync 3000") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("alter database db1 wal 1") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + tdSql.checkData(i, wal_index, 1) + + tdSql.execute("alter database db1 wal 2") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("alter database db1 fsync 0") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 0) + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("alter database db1 fsync 3000") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("alter database db1 fsync 18000") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 18000) + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("alter database db1 wal 1 fsync 3000") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + tdSql.checkData(i, wal_index, 1) + + tdSql.execute("drop database db1 ") + + @property + def fsync_create_err(self): + return [ + "create database db1 wal 0", + "create database db1 wal 3", + "create database db1 wal null", + "create database db1 wal true", + "create database db1 wal 1.1", + "create database db1 fsync -1", + "create database db1 fsync 180001", + "create database db1 fsync 10.111", + "create database db1 fsync true", + ] + + @property + def fsync_alter_err(self): + return [ + "alter database db1 wal 0", + "alter database db1 wal 3", + "alter database db1 wal null", + "alter database db1 wal true", + "alter database db1 wal 1.1", + "alter database db1 fsync -1", + "alter database db1 fsync 180001", + "alter database db1 fsync 10.111", + "alter database db1 fsync true", + ] + + def test_fsync_err(self): + for sql in self.fsync_create_err: + tdSql.error(sql) + tdSql.query("create database db1") + for sql in self.fsync_alter_err: + tdSql.error(sql) + tdSql.query("drop database db1") + + def all_test(self): + self.test_fsync_err() + self.test_fsync_current() + + # def __create_tb(self): + + # tdLog.printNoPrefix("==========step1:create table") + # create_stb_sql = f'''create table stb1( + # ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + # {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + # {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + # ) tags (t1 int) + # ''' + # create_ntb_sql = f'''create table t1( + # ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + # {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + # {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + # ) + # ''' + # tdSql.execute(create_stb_sql) + # tdSql.execute(create_ntb_sql) + + # for i in range(4): + # tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + # { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2} + + # def __insert_data(self, rows): + # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + # for i in range(rows): + # tdSql.execute( + # f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + # ) + # tdSql.execute( + # f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + # ) + # tdSql.execute( + # f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + # ) + # tdSql.execute( + # f'''insert into ct1 values + # ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) + # ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) + # ''' + # ) + + # tdSql.execute( + # f'''insert into ct4 values + # ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( + # { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, + # { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} + # ) + # ( + # { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, + # { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} + # ) + # ''' + # ) + + # tdSql.execute( + # f'''insert into ct2 values + # ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( + # { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, + # { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + # ) + # ( + # { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, + # { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + # ) + # ''' + # ) + + # for i in range(rows): + # insert_data = f'''insert into t1 values + # ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, + # "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) + # ''' + # tdSql.execute(insert_data) + # tdSql.execute( + # f'''insert into t1 values + # ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, + # { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, + # "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + # ) + # ( + # { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, + # { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, + # "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + # ) + # ''' + # ) + + def run(self): + tdSql.prepare() + + self.all_test() + + tdDnodes.stop(1) + tdDnodes.start(1) + + # tdSql.execute("use db") + + tdLog.printNoPrefix("==========step4:after wal, all check again ") + self.all_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index af3245df3d937cc2a4c4723c52a34fdd3ebeb561..679b41509891d1efe92507a81f7add51b9f76253 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 48058af295b5da8664a8a477803c7c9d8d3c526f..4b5be79a70e0b8a7c635b49bc8159a907d688a19 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -1,7 +1,9 @@ +from tabnanny import check import taos -import sys +import time import inspect import traceback +from dataclasses import dataclass from util.log import * from util.sql import * @@ -12,6 +14,10 @@ PRIVILEGES_ALL = "ALL" PRIVILEGES_READ = "READ" PRIVILEGES_WRITE = "WRITE" +WEIGHT_ALL = 5 +WEIGHT_READ = 2 +WEIGHT_WRITE = 3 + PRIMARY_COL = "ts" INT_COL = "c1" @@ -94,6 +100,7 @@ class TDconnect: self.cursor.close() self._conn.close() + def taos_connect( host = "127.0.0.1", port = 6030, @@ -111,6 +118,15 @@ def taos_connect( config=config ) + +@dataclass +class User: + name : str = None + passwd : str = None + db_set : set = None + priv : str = None + priv_weight : int = 0 + class TDTestCase: def init(self, conn, logSql): @@ -121,6 +137,22 @@ class TDTestCase: def __user_list(self): return [f"user_test{i}" for i in range(self.users_count) ] + def __users(self): + self.users = [] + self.root_user = User() + self.root_user.name = "root" + self.root_user.passwd = "taosdata" + self.root_user.db_set = set("*") + self.root_user.priv = PRIVILEGES_ALL + self.root_user.priv_weight = WEIGHT_ALL + for i in range(self.users_count): + user = User() + user.name = f"user_test{i}" + user.passwd = f"taosdata{i}" + user.db_set = set() + self.users.append(user) + return self.users + @property def __passwd_list(self): return [f"taosdata{i}" for i in range(self.users_count) ] @@ -205,76 +237,168 @@ class TDTestCase: def __grant_user_privileges(self, privilege, dbname=None, user_name="root"): return f"GRANT {privilege} ON {self.__priv_level(dbname)} TO {user_name} " - def grant_check(self, user="root", passwd="taosdata", priv=PRIVILEGES_ALL): - with taos_connect(user=user, passwd=passwd) as user: - user.query("use db") - user.query("show tables") - if priv in [PRIVILEGES_ALL, PRIVILEGES_READ]: - user.query("select * from ct1") - else: - user.error("select * from ct1") - if priv in [PRIVILEGES_ALL, PRIVILEGES_WRITE]: - user.query("insert into t1 (ts) values (now())") - else: - user.error("insert into t1 (ts) values (now())") - - def test_grant_current(self): - tdLog.printNoPrefix("==========step 1.0: if do not grant, can not read/write") - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=None) - - tdLog.printNoPrefix("==========step 1.1: grant read, can read, can not write") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_READ, user_name=self.__user_list[0]) + def __revoke_user_privileges(self, privilege, dbname=None, user_name="root"): + return f"REVOKE {privilege} ON {self.__priv_level(dbname)} FROM {user_name} " + + def __user_check(self, user:User=None, check_priv=PRIVILEGES_ALL): + if user is None: + user = self.root_user + with taos_connect(user=user.name, passwd=user.passwd) as use: + use.query("use db") + use.query("show tables") + if check_priv == PRIVILEGES_ALL: + use.query("select * from ct1") + use.query("insert into t1 (ts) values (now())") + elif check_priv == PRIVILEGES_READ: + use.query("select * from ct1") + use.error("insert into t1 (ts) values (now())") + elif check_priv == PRIVILEGES_WRITE: + use.error("select * from ct1") + use.query("insert into t1 (ts) values (now())") + elif check_priv is None: + use.error("select * from ct1") + use.error("insert into t1 (ts) values (now())") + + def __change_user_priv(self, user: User, pre_priv, invoke=False): + if user.priv == pre_priv and invoke : + return + if user.name == "root": + return + + if pre_priv.upper() == PRIVILEGES_ALL: + pre_weight = -5 if invoke else 5 + elif pre_priv.upper() == PRIVILEGES_READ: + pre_weight = -2 if invoke else 2 + elif pre_priv.upper() == PRIVILEGES_WRITE: + pre_weight = -3 if invoke else 3 + else: + return + pre_weight += user.priv_weight + + if pre_weight >= 5: + user.priv = PRIVILEGES_ALL + user.priv_weight = 5 + elif pre_weight == 3: + user.priv = PRIVILEGES_WRITE + user.priv_weight = pre_weight + elif pre_weight == 2: + user.priv_weight = pre_weight + user.priv = PRIVILEGES_READ + elif pre_weight in [1, -1]: + return + elif pre_weight <= 0: + user.priv_weight = 0 + user.priv = "" + + return user + + def grant_user(self, user: User = None, priv=PRIVILEGES_ALL, dbname=None): + if not user: + user = self.root_user + sql = self.__grant_user_privileges(privilege=priv, dbname=dbname, user_name=user.name) tdLog.info(sql) + if (user not in self.users and user.name != "root") or priv not in (PRIVILEGES_ALL, PRIVILEGES_READ, PRIVILEGES_WRITE): + tdSql.error(sql) tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_READ) + self.__change_user_priv(user=user, pre_priv=priv) + user.db_set.add(dbname) + time.sleep(1) - tdLog.printNoPrefix("==========step 1.2: grant write, can write, can not read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_WRITE, user_name=self.__user_list[1]) + def revoke_user(self, user: User = None, priv=PRIVILEGES_ALL, dbname=None): + sql = self.__revoke_user_privileges(privilege=priv, dbname=dbname, user_name=user.name) tdLog.info(sql) + if user is None or priv not in (PRIVILEGES_ALL, PRIVILEGES_READ, PRIVILEGES_WRITE): + tdSql.error(sql) tdSql.query(sql) - self.grant_check(user=self.__user_list[1], passwd=self.__passwd_list[1], priv=PRIVILEGES_WRITE) + self.__change_user_priv(user=user, pre_priv=priv, invoke=True) + if user.name != "root": + user.db_set.discard(dbname) if dbname else user.db_set.clear() + time.sleep(1) + + def test_priv_change_current(self): + tdLog.printNoPrefix("==========step 1.0: if do not grant, can not read/write") + self.__user_check(user=self.root_user) + self.__user_check(user=self.users[0], check_priv=None) + + tdLog.printNoPrefix("==========step 1.1: grant read, can read, can not write") + self.grant_user(user=self.users[0], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_READ) + + tdLog.printNoPrefix("==========step 1.2: grant write, can write") + self.grant_user(user=self.users[1], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_WRITE) tdLog.printNoPrefix("==========step 1.3: grant all, can write and read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_ALL, user_name=self.__user_list[2]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[2], passwd=self.__passwd_list[2], priv=PRIVILEGES_ALL) + self.grant_user(user=self.users[2]) + self.__user_check(user=self.users[2], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.4: change grant read to write, can write , can not read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_WRITE, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_WRITE) + tdLog.printNoPrefix("==========step 1.4: grant read to write = all ") + self.grant_user(user=self.users[0], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.5: change grant write to read, can not write , can read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_READ, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_READ) + tdLog.printNoPrefix("==========step 1.5: revoke write from all = read ") + self.revoke_user(user=self.users[0], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_READ) - tdLog.printNoPrefix("==========step 1.6: change grant read to all, can write , can read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_ALL, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_ALL) + tdLog.printNoPrefix("==========step 1.6: grant write to read = all") + self.grant_user(user=self.users[1], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.7: change grant all to write, can write , can not read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_WRITE, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_WRITE) + tdLog.printNoPrefix("==========step 1.7: revoke read from all = write ") + self.revoke_user(user=self.users[1], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_WRITE) - tdLog.printNoPrefix("==========step 1.8: change grant write to all, can write , can read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_ALL, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_ALL) + tdLog.printNoPrefix("==========step 1.8: grant read to all = all") + self.grant_user(user=self.users[0], priv=PRIVILEGES_ALL) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.9: change grant all to read, can not write , can read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_READ, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_READ) + tdLog.printNoPrefix("==========step 1.9: grant write to all = all") + self.grant_user(user=self.users[1], priv=PRIVILEGES_ALL) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_ALL) + + tdLog.printNoPrefix("==========step 1.10: grant all to read = all") + self.grant_user(user=self.users[0], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_ALL) + + tdLog.printNoPrefix("==========step 1.11: grant all to write = all") + self.grant_user(user=self.users[1], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_ALL) + + ### init user + self.revoke_user(user=self.users[0], priv=PRIVILEGES_WRITE) + self.revoke_user(user=self.users[1], priv=PRIVILEGES_READ) + + tdLog.printNoPrefix("==========step 1.12: revoke read from write = no change") + self.revoke_user(user=self.users[1], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_WRITE) + + tdLog.printNoPrefix("==========step 1.13: revoke write from read = no change") + self.revoke_user(user=self.users[0], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_READ) + + tdLog.printNoPrefix("==========step 1.14: revoke read from read = nothing") + self.revoke_user(user=self.users[0], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[0], check_priv=None) + + tdLog.printNoPrefix("==========step 1.15: revoke write from write = nothing") + self.revoke_user(user=self.users[1], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[1], check_priv=None) + + ### init user + self.grant_user(user=self.users[0], priv=PRIVILEGES_READ) + self.revoke_user(user=self.users[1], priv=PRIVILEGES_WRITE) + + tdLog.printNoPrefix("==========step 1.16: revoke all from write = nothing") + self.revoke_user(user=self.users[1], priv=PRIVILEGES_ALL) + self.__user_check(user=self.users[1], check_priv=None) + + tdLog.printNoPrefix("==========step 1.17: revoke all from read = nothing") + self.revoke_user(user=self.users[0], priv=PRIVILEGES_ALL) + self.__user_check(user=self.users[0], check_priv=None) + + tdLog.printNoPrefix("==========step 1.18: revoke all from all = nothing") + self.revoke_user(user=self.users[2], priv=PRIVILEGES_ALL) + self.__user_check(user=self.users[2], check_priv=None) def __grant_err(self): return [ @@ -288,13 +412,30 @@ class TDTestCase: f"GRANT {self.__privilege[0]} ON db.t1 TO {self.__user_list[0]}" , ] + def __revoke_err(self): + return [ + self.__revoke_user_privileges(privilege=self.__privilege[0], user_name="") , + self.__revoke_user_privileges(privilege=self.__privilege[0], user_name="*") , + self.__revoke_user_privileges(privilege=self.__privilege[1], dbname="not_exist_db", user_name=self.__user_list[0]), + self.__revoke_user_privileges(privilege="any_priv", user_name=self.__user_list[0]), + self.__revoke_user_privileges(privilege="", dbname="db", user_name=self.__user_list[0]) , + self.__revoke_user_privileges(privilege=" ".join(self.__privilege), user_name=self.__user_list[0]) , + f"REVOKE {self.__privilege[0]} ON * FROM {self.__user_list[0]}" , + f"REVOKE {self.__privilege[0]} ON db.t1 FROM {self.__user_list[0]}" , + ] + def test_grant_err(self): for sql in self.__grant_err(): tdSql.error(sql) - def test_grant(self): + def test_revoke_err(self): + for sql in self.__revoke_err(): + tdSql.error(sql) + + def test_change_priv(self): self.test_grant_err() - self.test_grant_current() + self.test_revoke_err() + self.test_priv_change_current() def test_user_create(self): self.create_user_current() @@ -455,7 +596,9 @@ class TDTestCase: tdSql.prepare() self.__create_tb() self.rows = 10 + self.users_count = 5 self.__insert_data(self.rows) + self.users = self.__users() tdDnodes.stop(1) tdDnodes.start(1) @@ -469,7 +612,6 @@ class TDTestCase: # root用户权限 # 创建用户测试 tdLog.printNoPrefix("==========step1: create user test") - self.users_count = 5 self.test_user_create() # 查看用户 @@ -482,7 +624,7 @@ class TDTestCase: self.login_err(self.__user_list[0], f"new{self.__passwd_list[0]}") # 用户权限设置 - self.test_grant() + self.test_change_priv() # 修改密码 tdLog.printNoPrefix("==========step3: alter user pass test") diff --git a/tests/system-test/7-tmq/subscribeStb.py b/tests/system-test/7-tmq/subscribeStb.py index 6fcc2d5e5fab3bff2bb6ff295dda242a43f52b98..d0463be710e77e7bf4f9f3d35e66a2acddee42a3 100644 --- a/tests/system-test/7-tmq/subscribeStb.py +++ b/tests/system-test/7-tmq/subscribeStb.py @@ -1377,17 +1377,17 @@ class TDTestCase: self.tmqCase1(cfgPath, buildPath) self.tmqCase2(cfgPath, buildPath) - #self.tmqCase3(cfgPath, buildPath) + self.tmqCase3(cfgPath, buildPath) self.tmqCase4(cfgPath, buildPath) self.tmqCase5(cfgPath, buildPath) - self.tmqCase6(cfgPath, buildPath) - self.tmqCase7(cfgPath, buildPath) - self.tmqCase8(cfgPath, buildPath) - self.tmqCase9(cfgPath, buildPath) - self.tmqCase10(cfgPath, buildPath) - self.tmqCase11(cfgPath, buildPath) - self.tmqCase12(cfgPath, buildPath) - self.tmqCase13(cfgPath, buildPath) + #self.tmqCase6(cfgPath, buildPath) + #self.tmqCase7(cfgPath, buildPath) + #self.tmqCase8(cfgPath, buildPath) + #self.tmqCase9(cfgPath, buildPath) + #self.tmqCase10(cfgPath, buildPath) + #self.tmqCase11(cfgPath, buildPath) + #self.tmqCase12(cfgPath, buildPath) + #self.tmqCase13(cfgPath, buildPath) def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/subscribeStb1.py b/tests/system-test/7-tmq/subscribeStb1.py new file mode 100644 index 0000000000000000000000000000000000000000..049b297d2df0f54e7d681c6236d942340da2d19f --- /dev/null +++ b/tests/system-test/7-tmq/subscribeStb1.py @@ -0,0 +1,1399 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class actionType(Enum): + CREATE_DATABASE = 0 + CREATE_STABLE = 1 + CREATE_CTABLE = 2 + INSERT_DATA = 3 + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def initConsumerInfoTable(self,cdbName='cdb'): + tdLog.info("drop consumeinfo table") + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + shellCmd = 'nohup ' + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, dbName,stbName): + tsql.execute("create table if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(dbName, stbName)) + tdLog.debug("complete to create %s.%s" %(dbName, stbName)) + return + + def create_ctables(self,tsql, dbName,stbName,ctbNum): + tsql.execute("use %s" %dbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfSql = 0 + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + rowsOfSql += 1 + if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsOfSql = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + + if parameterDict["actionType"] == actionType.CREATE_DATABASE: + self.create_database(tsql, parameterDict["dbName"]) + elif parameterDict["actionType"] == actionType.CREATE_STABLE: + self.create_stable(tsql, parameterDict["dbName"], parameterDict["stbName"]) + elif parameterDict["actionType"] == actionType.CREATE_CTABLE: + self.create_ctables(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + elif parameterDict["actionType"] == actionType.INSERT_DATA: + self.insert_data(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],parameterDict["batchNum"]) + else: + tdLog.exit("not support's action: ", parameterDict["actionType"]) + + return + + def tmqCase1(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 1: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db1', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 100 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + time.sleep(5) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 2: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db2', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + parameterDict2 = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db2', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict2['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_stable(tdSql, parameterDict2["dbName"], parameterDict2["stbName"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 100 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start create child tables of stb1 and stb2") + parameterDict['actionType'] = actionType.CREATE_CTABLE + parameterDict2['actionType'] = actionType.CREATE_CTABLE + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() + + prepareEnvThread.join() + prepareEnvThread2.join() + + tdLog.info("start insert data into child tables of stb1 and stb2") + parameterDict['actionType'] = actionType.INSERT_DATA + parameterDict2['actionType'] = actionType.INSERT_DATA + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() + + prepareEnvThread.join() + prepareEnvThread2.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def tmqCase3(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 3: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db3', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 13, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,parameterDict["dbName"],parameterDict["stbName"],parameterDict["ctbNum"],parameterDict["rowsPerTbl"],parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + time.sleep(2) + tdLog.info("drop som child table of stb1") + dropTblNum = 4 + tdSql.query("drop table if exists %s.%s_9"%(parameterDict["dbName"], parameterDict["stbName"])) + tdSql.query("drop table if exists %s.%s_8"%(parameterDict["dbName"], parameterDict["stbName"])) + tdSql.query("drop table if exists %s.%s_7"%(parameterDict["dbName"], parameterDict["stbName"])) + tdSql.query("drop table if exists %s.%s_3"%(parameterDict["dbName"], parameterDict["stbName"])) + + tdLog.info("drop some child tables, then start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + remaindrowcnt = parameterDict["rowsPerTbl"] * (parameterDict["ctbNum"] - dropTblNum) + + if not (totalConsumeRows < expectrowcnt and totalConsumeRows > remaindrowcnt): + tdLog.info("act consume rows: %d, expect consume rows: between %d and %d"%(totalConsumeRows, remaindrowcnt, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 3 end ...... ") + + def tmqCase4(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 4: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db4', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt/4: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) + tdLog.exit("tmq consume rows error!") + + self.initConsumerInfoTable() + consumerId = 1 + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("again start consume processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("again check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 4 end ...... ") + + def tmqCase5(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 5: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db5', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt/4: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) + tdLog.exit("tmq consume rows error!") + + self.initConsumerInfoTable() + consumerId = 1 + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("again start consume processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("again check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != (expectrowcnt * (1 + 1/4)): + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 5 end ...... ") + + def tmqCase6(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 6: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db6', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt/4: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) + tdLog.exit("tmq consume rows error!") + + self.initConsumerInfoTable() + consumerId = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:latest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("again start consume processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("again check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 6 end ...... ") + + def tmqCase7(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 7: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db7', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:latest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != 0: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + tdLog.exit("tmq consume rows error!") + + self.initConsumerInfoTable() + consumerId = 1 + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("again start consume processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("again check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != 0: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 7 end ...... ") + + def tmqCase8(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 8: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db8', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:latest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume 0 processor") + pollDelay = 10 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume 0 result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != 0: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + tdLog.exit("tmq consume rows error!") + + tdLog.info("start consume 1 processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start one new thread to insert data") + parameterDict['actionType'] = actionType.INSERT_DATA + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + prepareEnvThread.join() + + tdLog.info("start to check consume 0 and 1 result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdLog.info("start consume 2 processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start one new thread to insert data") + parameterDict['actionType'] = actionType.INSERT_DATA + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + prepareEnvThread.join() + + tdLog.info("start to check consume 0 and 1 and 2 result") + expectRows = 3 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt*2: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 8 end ...... ") + + def tmqCase9(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 9: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db9', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:latest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume 0 processor") + pollDelay = 10 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume 0 result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != 0: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + tdLog.exit("tmq consume rows error!") + + tdLog.info("start consume 1 processor") + self.initConsumerInfoTable() + consumerId = 1 + ifManualCommit = 0 + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start one new thread to insert data") + parameterDict['actionType'] = actionType.INSERT_DATA + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + prepareEnvThread.join() + + tdLog.info("start to check consume 0 and 1 result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdLog.info("start consume 2 processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start one new thread to insert data") + parameterDict['actionType'] = actionType.INSERT_DATA + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + prepareEnvThread.join() + + tdLog.info("start to check consume 0 and 1 and 2 result") + expectRows = 3 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt*2: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 9 end ...... ") + + def tmqCase10(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 10: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db10', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:latest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume 0 processor") + pollDelay = 10 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume 0 result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != 0: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + tdLog.exit("tmq consume rows error!") + + tdLog.info("start consume 1 processor") + self.initConsumerInfoTable() + consumerId = 1 + ifManualCommit = 1 + self.insertConsumerInfo(consumerId, expectrowcnt-10000,topicList,keyList,ifcheckdata,ifManualCommit) + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start one new thread to insert data") + parameterDict['actionType'] = actionType.INSERT_DATA + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + prepareEnvThread.join() + + tdLog.info("start to check consume 0 and 1 result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt-10000: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt-10000)) + tdLog.exit("tmq consume rows error!") + + tdLog.info("start consume 2 processor") + self.initConsumerInfoTable() + consumerId = 2 + ifManualCommit = 1 + self.insertConsumerInfo(consumerId, expectrowcnt+10000,topicList,keyList,ifcheckdata,ifManualCommit) + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start one new thread to insert data") + parameterDict['actionType'] = actionType.INSERT_DATA + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + prepareEnvThread.join() + + tdLog.info("start to check consume 0 and 1 and 2 result") + expectRows = 3 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt*2: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 10 end ...... ") + + def tmqCase11(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 11: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db11', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:none' + self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != 0: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + tdLog.exit("tmq consume rows error!") + + self.initConsumerInfoTable() + consumerId = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:none' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("again start consume processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("again check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != 0: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, 0)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 11 end ...... ") + + def tmqCase12(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 12: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db12', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt/4: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) + tdLog.exit("tmq consume rows error!") + + self.initConsumerInfoTable() + consumerId = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:none' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("again start consume processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("again check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt/4: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 12 end ...... ") + + def tmqCase13(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 13: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db13', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt/4: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) + tdLog.exit("tmq consume rows error!") + + self.initConsumerInfoTable() + consumerId = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:none' + self.insertConsumerInfo(consumerId, expectrowcnt/2,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("again start consume processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("again check consume result") + expectRows = 2 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt*(1/2+1/4): + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*(1/2+1/4))) + tdLog.exit("tmq consume rows error!") + + self.initConsumerInfoTable() + consumerId = 2 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:none' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("again start consume processor") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("again check consume result") + expectRows = 3 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 13 end ...... ") + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + #self.tmqCase1(cfgPath, buildPath) + #self.tmqCase2(cfgPath, buildPath) + #self.tmqCase3(cfgPath, buildPath) + #self.tmqCase4(cfgPath, buildPath) + #self.tmqCase5(cfgPath, buildPath) + self.tmqCase6(cfgPath, buildPath) + self.tmqCase7(cfgPath, buildPath) + self.tmqCase8(cfgPath, buildPath) + self.tmqCase9(cfgPath, buildPath) + self.tmqCase10(cfgPath, buildPath) + self.tmqCase11(cfgPath, buildPath) + self.tmqCase12(cfgPath, buildPath) + self.tmqCase13(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 06be29a6362b091d24867288096569b8a9732bda..a6dfd5129fd9beed3ef7e649713105854470cb5e 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -9,8 +9,8 @@ python3 ./test.py -f 0-others/telemetry.py python3 ./test.py -f 0-others/taosdMonitor.py python3 ./test.py -f 0-others/udfTest.py -# TODO privilege has error -# python3 ./test.py -f 0-others/user_control.py +python3 ./test.py -f 0-others/user_control.py +python3 ./test.py -f 0-others/fsync.py #python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py @@ -65,3 +65,4 @@ python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py python3 ./test.py -f 7-tmq/subscribeDb1.py python3 ./test.py -f 7-tmq/subscribeStb.py +python3 ./test.py -f 7-tmq/subscribeStb1.py diff --git a/tests/test/c/createTable.c b/tests/test/c/createTable.c index c53ae0136cb2da84fd35634d0011a18a36d57c67..6a0f8e244ebcce79ac60a353df5d5343d58b4b34 100644 --- a/tests/test/c/createTable.c +++ b/tests/test/c/createTable.c @@ -261,7 +261,7 @@ void *threadFunc(void *param) { int64_t startTs = taosGetTimestampUs(); TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); - if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { + if (code != 0) { pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code)); } taos_free_result(pRes); diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 96d7741897d2dd3d888585a4ed5a12be53f5c72f..5a972e071a5e28708cfb561ef1015d72ba830219 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -233,7 +233,6 @@ int64_t getDirectorySize(char* dir) { int queryDB(TAOS* taos, char* command) { TAOS_RES* pRes = taos_query(taos, command); int code = taos_errno(pRes); - // if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { if (code != 0) { pError("failed to reason:%s, sql: %s", tstrerror(code), command); taos_free_result(pRes); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index ab54c819cfe81cee3f5d10c0622478b0e44445b5..e0f58d052f612fca7ad0a257d8e137d8d4a5a1f6 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -258,7 +258,6 @@ static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) int queryDB(TAOS* taos, char* command) { TAOS_RES* pRes = taos_query(taos, command); int code = taos_errno(pRes); - // if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { if (code != 0) { pError("failed to reason:%s, sql: %s", tstrerror(code), command); taos_free_result(pRes); diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 969332ba5f032efcc2098c3bcc215456b8e6a509..1c751f290a2a23c1496cdbe01f90a3cb9985c860 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -98,6 +98,8 @@ SScript *simProcessCallOver(SScript *script) { return NULL; } + if (simScriptPos == -1) return NULL; + return simScriptList[simScriptPos]; } else { simDebug("script:%s, is stopped", script->fileName); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d6c295a2229a8c471ba119ddde87d79f5fe4370c..a866488d3ad2d239c47b3279f506e755737b88bf 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -742,6 +742,7 @@ void shellReadHistory() { int32_t read_size = 0; while ((read_size = taosGetLineFile(pFile, &line)) != -1) { line[read_size - 1] = '\0'; + taosMemoryFree(pHistory->hist[pHistory->hend]); pHistory->hist[pHistory->hend] = strdup(line); pHistory->hend = (pHistory->hend + 1) % SHELL_MAX_HISTORY_SIZE; @@ -763,7 +764,8 @@ void shellWriteHistory() { for (int32_t i = pHistory->hstart; i != pHistory->hend;) { if (pHistory->hist[i] != NULL) { taosFprintfFile(pFile, "%s\n", pHistory->hist[i]); - taosMemoryFreeClear(pHistory->hist[i]); + taosMemoryFree(pHistory->hist[i]); + pHistory->hist[i] = NULL; } i = (i + 1) % SHELL_MAX_HISTORY_SIZE; } @@ -771,6 +773,16 @@ void shellWriteHistory() { taosCloseFile(&pFile); } +void shellCleanupHistory() { + SShellHistory *pHistory = &shell.history; + for (int32_t i = 0; i < SHELL_MAX_HISTORY_SIZE; ++i) { + if (pHistory->hist[i] != NULL) { + taosMemoryFree(pHistory->hist[i]); + pHistory->hist[i] = NULL; + } + } +} + void shellPrintError(TAOS_RES *tres, int64_t st) { int64_t et = taosGetTimestampUs(); fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6); @@ -971,6 +983,7 @@ int32_t shellExecute() { taos_close(shell.conn); shellWriteHistory(); + shellCleanupHistory(); return 0; } @@ -996,5 +1009,6 @@ int32_t shellExecute() { taosThreadClear(&shell.pid); } + shellCleanupHistory(); return 0; }